From 62fec8cfb082c054f360f096993a64af713f50cf Mon Sep 17 00:00:00 2001 From: Pascal Christoph Date: Fri, 25 Oct 2024 16:44:22 +0200 Subject: [PATCH] Add script to generate a dot from flux-commands.md (#498) WIP - add a dot, generated by the script, in part manually altered - add svg generated from dot --- docs/flux/flux-commands.dot | 130 ++++++ docs/flux/flux-commands.svg | 854 ++++++++++++++++++++++++++++++++++++ docs/flux/makeGraph.sh | 43 ++ 3 files changed, 1027 insertions(+) create mode 100644 docs/flux/flux-commands.dot create mode 100644 docs/flux/flux-commands.svg create mode 100644 docs/flux/makeGraph.sh diff --git a/docs/flux/flux-commands.dot b/docs/flux/flux-commands.dot new file mode 100644 index 0000000..a00388e --- /dev/null +++ b/docs/flux/flux-commands.dot @@ -0,0 +1,130 @@ +digraph G { + { + node [margin=0 fontcolor=black fontsize=32 width=0.5 shape=oval style=filled] + String [fillcolor=yellow fixedsize=false fillcolor="yellow", label=String] + Reader [fillcolor=yellow fixedsize=false fillcolor="yellow", label=Reader] + Object [fillcolor=yellow fixedsize=false fillcolor="yellow", label=Object] + Map [fillcolor=yellow fixedsize=false fillcolor="yellow", label=Map] + Triple [fillcolor=yellow fixedsize=false fillcolor="yellow", label=Triple] + Void [fillcolor=yellow fixedsize=false fillcolor="yellow", label=Void] + ListMap [fillcolor=yellow fixedsize=false fillcolor="yellow", label=ListMap] + Document [fillcolor=yellow fixedsize=false fillcolor="yellow", label=Document] + StreamReceiver [fillcolor=yellow fixedsize=false fillcolor="yellow", label=StreamReceiver] + XmlReceiver [fillcolor=yellow fixedsize=false fillcolor="yellow", label=XmlReceiver] + } + rankdir=LR + concentrate=true + + StreamReceiver -> StreamReceiver [fillcolor="red", label="add-oreaggregation"]; + String -> String [fillcolor="red", label="add-preamble-epilogue"]; + Reader -> String [fillcolor="red", label="as-formeta-records"]; + Reader -> String [fillcolor="red", label="as-lines"]; + Reader -> String [fillcolor="red", label="as-records"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="batch-log"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="batch-reset"]; + Triple -> Triple [fillcolor="red", label="calculate-metrics"]; + Object -> Object [fillcolor="red", label="catch-object-exception"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="catch-stream-exception"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="change-id"]; + Triple -> StreamReceiver [fillcolor="red", label="collect-triples"]; + Triple -> Triple [fillcolor="red", label="count-triples"]; + String -> StreamReceiver [fillcolor="red", label="decode-aseq"]; + String -> StreamReceiver [fillcolor="red", label="decode-csv"]; + String -> StreamReceiver [fillcolor="red", label="decode-formeta"]; + Reader -> StreamReceiver [fillcolor="red", label="decode-html"]; + String -> StreamReceiver [fillcolor="red", label="decode-json"]; + String -> StreamReceiver [fillcolor="red", label="decode-mab"]; + String -> StreamReceiver [fillcolor="red", label="decode-marc21"]; + String -> StreamReceiver [fillcolor="red", label="decode-pica"]; + String -> String [fillcolor="red", label="decode-string"]; + Reader -> XmlReceiver [fillcolor="red", label="decode-xml"]; + String -> StreamReceiver [fillcolor="red", label="decode-yaml"]; + Object -> Object [fillcolor="red", label="decouple"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="defer-stream"]; + String -> Triple [fillcolor="red", label="digest-file"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="discard-events"]; + Object -> Object [fillcolor="red", label="draw-uniform-sample"]; + StreamReceiver -> String [fillcolor="red", label="encode-csv"]; + StreamReceiver -> String [fillcolor="red", label="encode-formeta"]; + StreamReceiver -> String [fillcolor="red", label="encode-json"]; + StreamReceiver -> String [fillcolor="red", label="encode-literals"]; + StreamReceiver -> String [fillcolor="red", label="encode-marc21"]; + StreamReceiver -> String [fillcolor="red", label="encode-marcxml"]; + StreamReceiver -> String [fillcolor="red", label="encode-pica"]; + StreamReceiver -> String [fillcolor="red", label="encode-xml"]; + StreamReceiver -> String [fillcolor="red", label="encode-yaml"]; + Reader -> String [fillcolor="red", label="extract-element"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="filter"]; + Object -> Object [fillcolor="red", label="filter-duplicate-objects"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="filter-null-values"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="filter-records-by-path"]; + String -> String [fillcolor="red", label="filter-strings"]; + Triple -> Triple [fillcolor="red", label="filter-triples"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="fix"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="flatten"]; + Document -> StreamReceiver [fillcolor="red", label="from-jdom-document"]; + XmlReceiver -> StreamReceiver [fillcolor="red", label="handle-cg-xml"]; + XmlReceiver -> StreamReceiver [fillcolor="red", label="handle-comarcxml"]; + XmlReceiver -> StreamReceiver [fillcolor="red", label="handle-generic-xml"]; + XmlReceiver -> StreamReceiver [fillcolor="red", label="handle-mabxml"]; + XmlReceiver -> StreamReceiver [fillcolor="red", label="handle-marcxml"]; + XmlReceiver -> StreamReceiver [fillcolor="red", label="handle-picaxml"]; + Object -> Object [fillcolor="red", label="jscript"]; + String -> String [fillcolor="red", label="json-to-elasticsearch-bulk"]; + String -> String [fillcolor="red", label="lines-to-records"]; + StreamReceiver -> String [fillcolor="red", label="list-fix-paths"]; + StreamReceiver -> String [fillcolor="red", label="list-fix-values"]; + StreamReceiver -> String [fillcolor="red", label="literal-to-object"]; + Object -> Object [fillcolor="red", label="log-object"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="log-stream"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="log-stream-time"]; + Object -> Object [fillcolor="red", label="log-time"]; + Map -> StreamReceiver [fillcolor="red", label="map-to-stream"]; + String -> String [fillcolor="red", label="match"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="merge-batch-stream"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="merge-same-ids"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="morph"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="normalize-unicode-stream"]; + String -> String [fillcolor="red", label="normalize-unicode-string"]; + Object -> Object [fillcolor="red", label="object-batch-log"]; + Object -> Object [fillcolor="red", label="object-tee"]; + -> StreamReceiver [fillcolor="red", label="object-to-literal"]; + String -> Reader [fillcolor="red", label="open-file"]; + String -> Reader [fillcolor="red", label="open-http"]; + String -> Reader [fillcolor="red", label="open-oaipmh"]; + String -> Reader [fillcolor="red", label="open-resource"]; + Reader -> Reader [fillcolor="red", label="open-tar"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="pass-through"]; + Object -> Void [fillcolor="red", label="print"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="rdf-macros"]; + Reader -> StreamReceiver [fillcolor="red", label="read-beacon"]; + String -> String [fillcolor="red", label="read-dir"]; + String -> Reader [fillcolor="red", label="read-string"]; + String -> Triple [fillcolor="red", label="read-triples"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="record-to-entity"]; + String -> StreamReceiver [fillcolor="red", label="regex-decode"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="remodel-pica-multiscript"]; + Triple -> Triple [fillcolor="red", label="reorder-triple"]; + Object -> Object [fillcolor="red", label="reset-object-batch"]; + Triple -> Triple [fillcolor="red", label="retrieve-triple-objects"]; + Triple -> Triple [fillcolor="red", label="sort-triples"]; + String -> String [fillcolor="red", label="split-lines"]; + XmlReceiver -> StreamReceiver [fillcolor="red", label="split-xml-elements"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="stream-count"]; + StreamReceiver -> StreamReceiver [fillcolor="red", label="stream-tee"]; + StreamReceiver -> Triple [fillcolor="red", label="stream-to-triples"]; + StreamReceiver -> String [fillcolor="red", label="stream-to-xml"]; + ListMap -> StreamReceiver [fillcolor="red", label="string-list-map-to-stream"]; + Object -> String [fillcolor="red", label="template"]; + Object -> Object [fillcolor="red", label="thread-object-tee"]; + StreamReceiver -> Document [fillcolor="red", label="to-jdom-document"]; + Triple -> StreamReceiver [fillcolor="red", label="triples-to-stream"]; + String -> String [fillcolor="red", label="validate-json"]; + Object -> Object [fillcolor="red", label="wait-for-inputs"]; + Object -> Void [fillcolor="red", label="write"]; + Object -> Void [fillcolor="red", label="write-files"]; + Triple -> Void [fillcolor="red", label="write-triple-objects"]; + Triple -> Void [fillcolor="red", label="write-triples"]; + StreamReceiver -> Void [fillcolor="red", label="write-xml-files"]; + XmlReceiver -> XmlReceiver [fillcolor="red", label="xml-tee"]; +} diff --git a/docs/flux/flux-commands.svg b/docs/flux/flux-commands.svg new file mode 100644 index 0000000..c579903 --- /dev/null +++ b/docs/flux/flux-commands.svg @@ -0,0 +1,854 @@ + + + + + + +G + + + +String + +String + + + +String->String + + +add-preamble-epilogue + + + +String->String + + +decode-string + + + +String->String + + +filter-strings + + + +String->String + + +json-to-elasticsearch-bulk + + + +String->String + + +lines-to-records + + + +String->String + + +match + + + +String->String + + +normalize-unicode-string + + + +String->String + + +read-dir + + + +String->String + + +split-lines + + + +String->String + + +validate-json + + + +Reader + +Reader + + + +String->Reader + + +open-file + + + +String->Reader + + +open-http + + + +String->Reader + + +open-oaipmh + + + +String->Reader + + +open-resource + + + +String->Reader + + +read-string + + + +Triple + +Triple + + + +String->Triple + +digest-file + + + +String->Triple + + + + +read-triples + + + +StreamReceiver + +StreamReceiver + + + +String->StreamReceiver + +decode-aseq + + + +String->StreamReceiver + +decode-csv + + + +String->StreamReceiver + +decode-formeta + + + +String->StreamReceiver + +decode-json + + + +String->StreamReceiver + +decode-mab + + + +String->StreamReceiver + +decode-marc21 + + + +String->StreamReceiver + +decode-pica + + + +String->StreamReceiver + +decode-yaml + + + +String->StreamReceiver + + + + +regex-decode + + + +Reader->String + + +as-formeta-records + + + +Reader->String + + +as-lines + + + +Reader->String + + +as-records + + + +Reader->String + + +extract-element + + + +Reader->Reader + + +open-tar + + + +Reader->StreamReceiver + +decode-html + + + +Reader->StreamReceiver + + + + +read-beacon + + + +XmlReceiver + +XmlReceiver + + + +Reader->XmlReceiver + + +decode-xml + + + +Object + +Object + + + +Object->String + + +template + + + +Object->Object + + +catch-object-exception + + + +Object->Object + + +decouple + + + +Object->Object + + +draw-uniform-sample + + + +Object->Object + + +filter-duplicate-objects + + + +Object->Object + + +jscript + + + +Object->Object + + +log-object + + + +Object->Object + + +log-time + + + +Object->Object + + +object-batch-log + + + +Object->Object + + +object-tee + + + +Object->Object + + +reset-object-batch + + + +Object->Object + + +thread-object-tee + + + +Object->Object + + +wait-for-inputs + + + +Void + +Void + + + +Object->Void + +print + + + +Object->Void + +write + + + +Object->Void + + + + + +write-files + + + +Map + +Map + + + +Map->StreamReceiver + + +map-to-stream + + + +Triple->Triple + + +calculate-metrics + + + +Triple->Triple + + +count-triples + + + +Triple->Triple + + +filter-triples + + + +Triple->Triple + + +reorder-triple + + + +Triple->Triple + + +retrieve-triple-objects + + + +Triple->Triple + + +sort-triples + + + +Triple->Void + + +write-triple-objects + + + +Triple->Void + + +write-triples + + + +Triple->StreamReceiver + + +collect-triples + + + +Triple->StreamReceiver + + +triples-to-stream + + + +ListMap + +ListMap + + + +ListMap->StreamReceiver + + +string-list-map-to-stream + + + +Document + +Document + + + +Document->StreamReceiver + + +from-jdom-document + + + +StreamReceiver->String + +encode-csv + + + +StreamReceiver->String + +encode-formeta + + + +StreamReceiver->String + +encode-json + + + +StreamReceiver->String + +encode-literals + + + +StreamReceiver->String + +encode-marc21 + + + +StreamReceiver->String + +encode-marcxml + + + +StreamReceiver->String + +encode-pica + + + +StreamReceiver->String + + + + +encode-xml + + + +StreamReceiver->String + +encode-yaml + + + +StreamReceiver->String + + + +list-fix-paths + + + +StreamReceiver->String + +list-fix-values + + + +StreamReceiver->String + +literal-to-object + + + +StreamReceiver->String + + + + +stream-to-xml + + + +StreamReceiver->Triple + + +stream-to-triples + + + +StreamReceiver->Void + +write-xml-files + + + +StreamReceiver->Document + + +to-jdom-document + + + +StreamReceiver->StreamReceiver + + +add-oreaggregation + + + +StreamReceiver->StreamReceiver + + +batch-log + + + +StreamReceiver->StreamReceiver + + +batch-reset + + + +StreamReceiver->StreamReceiver + + +catch-stream-exception + + + +StreamReceiver->StreamReceiver + + +change-id + + + +StreamReceiver->StreamReceiver + + +defer-stream + + + +StreamReceiver->StreamReceiver + + +discard-events + + + +StreamReceiver->StreamReceiver + + +filter + + + +StreamReceiver->StreamReceiver + + +filter-null-values + + + +StreamReceiver->StreamReceiver + + +filter-records-by-path + + + +StreamReceiver->StreamReceiver + + +fix + + + +StreamReceiver->StreamReceiver + + +flatten + + + +StreamReceiver->StreamReceiver + + +log-stream + + + +StreamReceiver->StreamReceiver + + +log-stream-time + + + +StreamReceiver->StreamReceiver + + +merge-batch-stream + + + +StreamReceiver->StreamReceiver + + +merge-same-ids + + + +StreamReceiver->StreamReceiver + + +morph + + + +StreamReceiver->StreamReceiver + + +normalize-unicode-stream + + + +StreamReceiver->StreamReceiver + + +pass-through + + + +StreamReceiver->StreamReceiver + + +rdf-macros + + + +StreamReceiver->StreamReceiver + + +record-to-entity + + + +StreamReceiver->StreamReceiver + + +remodel-pica-multiscript + + + +StreamReceiver->StreamReceiver + + +stream-count + + + +StreamReceiver->StreamReceiver + + +stream-tee + + + +XmlReceiver->StreamReceiver + + +handle-cg-xml + + + +XmlReceiver->StreamReceiver + + +handle-comarcxml + + + +XmlReceiver->StreamReceiver + + +handle-generic-xml + + + +XmlReceiver->StreamReceiver + + +handle-mabxml + + + +XmlReceiver->StreamReceiver + + +handle-marcxml + + + +XmlReceiver->StreamReceiver + + +handle-picaxml + + + +XmlReceiver->StreamReceiver + + +split-xml-elements + + + +XmlReceiver->XmlReceiver + + +xml-tee + + + +unknown + +unknown + + + +unknown->StreamReceiver + + +object-to-literal + + + diff --git a/docs/flux/makeGraph.sh b/docs/flux/makeGraph.sh new file mode 100644 index 0000000..958af27 --- /dev/null +++ b/docs/flux/makeGraph.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Generates a dot graphviz file from flux-commands.md. +# See https://github.com/metafacture/metafacture-core/issues/498. +# It's a draft for there are some steps missing to generate the nice colored svg. +# +# Called like: $ bash makeGraph.sh > flux-commands.dot + +INPUT_FN="flux-commands.md" + +unset IFS + +# preprocessing +echo "digraph G { + { + node [margin=0 fontcolor=black fontsize=32 width=0.5 shape=oval style=filled] + String [fillcolor=yellow fixedsize=false label="String"] + Reader [fillcolor=yellow fixedsize=false label="Reader"] + Object [fillcolor=yellow fixedsize=false label="Object"] + Map [fillcolor=yellow fixedsize=false label="Map"] + Triple [fillcolor=yellow fixedsize=false label="Triple"] + Void [fillcolor=yellow fixedsize=false label="Void"] + ListMap [fillcolor=yellow fixedsize=false label="ListMap"] + Document [fillcolor=yellow fixedsize=false label="Document"] + StreamReceiver [fillcolor=yellow fixedsize=false label="StreamReceiver"] + XmlReceiver [fillcolor=yellow fixedsize=false label="XmlReceiver"] + } + rankdir=LR + concentrate=true +" + +sed '/./{:a;N;s/\n\(.\)/ \1/;ta}' $INPUT_FN |grep "signature:" > $INPUT_FN.tmp + +while read -r LINE; do + label=$(echo "$LINE" | sed 's#\(.*\) ---.*#\1#g') + nodes=$(echo "$LINE" | sed 's#.*signature:\(.* -> .*\) - .*#\1#g'|sed 's#\(.* -> .*\) - .*#\1#g') + printf "$nodes [label=\"$label\"];\n"; +done < $INPUT_FN.tmp +echo "}" + +exit + +dot -Tsvg flux-commands.dot > flux-commands.svg