xprocessor : add support for xprocessor which provides documentation …

…though --help-json

api/models/processor_doc.go

@@ -9,6 +9,7 @@ package models
 // swagger:model processorDoc
 type processorDoc struct {
 	Kind     string `json:"kind"`
+	Behavior string `json:"behavior"`
 	Name     string `json:"name"`
 	Doc      string `json:"doc"`
 	DocShort string `json:"doc_short"`

api/models/xprocessor.go

@@ -15,6 +15,7 @@ type XProcessor struct {
 	StdoutAs              string   `json:"stdout_as" mapstructure:"stdout_as"`
 	Description           string   `json:"description"`
 	OptionsCompositionTpl string   `json:"options_composition_tpl" mapstructure:"options_composition_tpl"`
+	HasDoc                bool     `json:"has_doc" mapstructure:"has_doc"`
 
 	CreatedAt time.Time
 	UpdatedAt time.Time

cmd/bitfanUI/assets/public/js/bitbar.js

@@ -118,7 +118,7 @@
                 } else if (result.length > 1) {
                     // multiple items found
                     // console.log("crazy ids found for " + id)
-                    return
+                    // return
                 }
 
                 if (result[0].help != null && result[0].help != "") {
@@ -143,18 +143,20 @@
                 // find the "onSelect" callback
                 var result = $.grep(this.citems, function(e) { return e.id == id; });
                 if (result.length == 0) {
-                    // console.log("id " + id + " not found")
-                } else if (result.length == 1) {
-                    if (isFunction(result[0].onSelect)) {
-                        this.context.selected.unshift(id)
-                        result[0].onSelect(this.context,result[0])
-                    } else {
-                        // console.log("no callback for item " + id)
-                    }
+                    return
+                } 
+
+                if (result.length > 1) {
+                    console.log("bitbar: multiple entry with the same key : " + id)
+                }
+
+                if (isFunction(result[0].onSelect)) {
+                    this.context.selected.unshift(id)
+                    result[0].onSelect(this.context,result[0])
                 } else {
-                    // multiple items found
-                    // console.log("crazy ids found for " + id)
+                    console.log("bitbar: no callback for item " + id)
                 }
+
             },
 
             new: bitbarNew,

cmd/bitfanUI/assets/public/js/playground.js

@@ -145,7 +145,7 @@ function bitfanOptionText(option, withDoc) {
     }
 
     var value = labelStr + " => ";
-
+    
     switch (option.Type) {
         case "hash":
             if (option.DefaultValue != null) {
@@ -162,7 +162,7 @@ function bitfanOptionText(option, withDoc) {
             }
             break;
         case "string":
-            if (option.DefaultValue != null) {
+            if (option.DefaultValue != null && option.DefaultValue != "" ) {
                 value += option.DefaultValue;
             } else {
                 value += '""';
@@ -247,7 +247,18 @@ function bitfanProcessorInsertTemplate(c) {
             }
             finalTxt = key + " {\n"
 
+            withDoc = false
+            if (c.selected[0] == "insert_full") {
+                withDoc = true
+            }
+
             var optionsLen = proc.Options.Options.length;
+            if (optionsLen == 0 && withDoc == true){
+                const regex = /\n/gm;
+                const subst = `  # `;
+                // The substituted value will be contained in the result variable
+                    finalTxt += "  # "+proc.Doc.replace(regex, subst)+"\n"              
+            }
             for (var i = 0; i < optionsLen; i++) {
                 let option = proc.Options.Options[i]
 
@@ -256,12 +267,7 @@ function bitfanProcessorInsertTemplate(c) {
                     continue
                 }
 
-                withDoc = false
-                if (c.selected[0] == "insert_full") {
-                    withDoc = true
-                }
                 finalTxt += bitfanOptionText(option, withDoc)
-
             }
             finalTxt = finalTxt + "}\n"
 
@@ -336,12 +342,36 @@ $(document).ready(function() {
                 if (key.startsWith("input") || key.startsWith("output")) {
                     labelStr = "doc " + key.replace("_", " ");
                 }
+
+                switch (processors_docs[key].Behavior) {
+                       case "producer":
+                            labelStr = "doc input " + key;
+                            break;
+                       case "transformer":
+                            labelStr = "doc filter " + key;
+                            break;
+                        case "consumer":
+                            labelStr = "doc output " + key;
+                            break;
+                }
+
                 bitbar.items.push({
                     onSelect: bitfanProcessorMenu,
                     id: key,
                     label: labelStr,
                     help: processors_docs[key].Doc,
                 })
+
+                if (processors_docs[key].Behavior == "producer") {   
+                    bitbar.items.push({
+                        onSelect: bitfanProcessorMenu,
+                        id: key,
+                        label: "doc filter " + key,
+                        help: processors_docs[key].Doc,
+                    })
+                }
+
+
             }
 
         },

cmd/bitfanUI/assets/views/xprocessors/_sidebar.html

@@ -50,6 +50,11 @@ <h2>PlayTest</h2>
     ">Stream events to I/O</label>
   </div>
 
+  <div class="form-group">
+    <input type="checkbox" name="has_doc" id="has_doc" value="true" {{if .xprocessor.HasDoc}}checked{{end}}>
+    <label for="has_doc" title="xprocessor provides documentation as json when started with --help-json">support --help-json</label>
+  </div>
+
 
 <div class="form-group">
     <label for="stdin_as">incoming event (stdin) received as </label>

cmd/bitfanUI/assets/views/xprocessors/edit.html

@@ -118,9 +118,11 @@ <h1>Edit xProcessor informations</h1>
    abc=>"Hello world"
    debug => ""
    "_" => "foo.html bar.html"
+   count => {"a"=>"1234","c"=>"lorem"}
+   issues => ["A","B"]
 }
 
-will be passed to command arguments as : --abc="Hello world" --debug foo.html bar.html
+will be passed to command arguments as : --abc="Hello world" --debug  --count="a:1234" --count="c:lorem" --issues="A" --issues="B" foo.html bar.html
 </pre>
 
     </small>

cmd/bitfanUI/server/server.go

@@ -231,7 +231,8 @@ func updateXProcessor(c *gin.Context) {
 	xprocessorUUID := c.Param("id")
 
 	var data = map[string]interface{}{
-		"stream": false,
+		"stream":  false,
+		"has_doc": false,
 	}
 	if _, ok := c.Request.PostForm["behavior"]; ok {
 		data["behavior"] = c.Request.PostFormValue("behavior")
@@ -245,6 +246,9 @@ func updateXProcessor(c *gin.Context) {
 	if _, ok := c.Request.PostForm["stream"]; ok {
 		data["stream"] = true
 	}
+	if _, ok := c.Request.PostForm["has_doc"]; ok {
+		data["has_doc"] = true
+	}
 	if _, ok := c.Request.PostForm["stdin_as"]; ok {
 		data["stdin_as"] = c.Request.PostFormValue("stdin_as")
 	}

core/core.go

@@ -11,6 +11,7 @@ import (
 	"github.com/vjeantet/bitfan/core/metrics"
 	"github.com/vjeantet/bitfan/core/webhook"
 	"github.com/vjeantet/bitfan/processors/doc"
+	"github.com/vjeantet/bitfan/processors/xprocessor"
 	"github.com/vjeantet/bitfan/store"
 )
 
@@ -217,10 +218,31 @@ func ProcessorsDocs(code string) map[string]*doc.Processor {
 			docs[code] = proc().Doc()
 		}
 
+		// get Xprocessors docs
+		if xp, err := Storage().FindOneXProcessorByName(code); err == nil {
+			docs[xp.Label] = xprocessor.NewWithSpec(&xp).Doc()
+			docs[xp.Label].Name = xp.Label
+			if docs[xp.Label].Doc == "" {
+				docs[xp.Label].Doc = xp.Description
+			}
+		}
+		return docs
 	} else {
 		for code, proc := range availableProcessorsFactory {
 			docs[code] = proc().Doc()
 		}
+		// findXprocessors
+		xprocessors := Storage().FindXProcessors("")
+		for _, xp := range xprocessors {
+			docs[xp.Label] = xprocessor.NewWithSpec(&xp).Doc()
+			docs[xp.Label].Name = xp.Label
+			if docs[xp.Label].Doc == "" {
+				docs[xp.Label].Doc = xp.Description
+			}
+		}
+
+		// get xProcessor doc
+		return docs
 	}
 
 	return docs

processors/doc/structs.go

@@ -45,6 +45,7 @@ type CodecOption struct {
 }
 
 type Processor struct {
+	Behavior   string
 	Name       string
 	ImportPath string
 	Doc        string

processors/xprocessor/xprocessor.go

@@ -2,6 +2,7 @@ package xprocessor
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"html/template"
 	"io"
@@ -17,6 +18,7 @@ import (
 	"github.com/vjeantet/bitfan/api/models"
 	"github.com/vjeantet/bitfan/codecs"
 	"github.com/vjeantet/bitfan/processors"
+	"github.com/vjeantet/bitfan/processors/doc"
 )
 
 func NewWithSpec(spec *models.XProcessor) processors.Processor {
@@ -35,10 +37,12 @@ func NewWithSpec(spec *models.XProcessor) processors.Processor {
 	if spec.Stream == true {
 		p := &streamProcessor{}
 		p.opt = opt
+		p.hasDoc = spec.HasDoc
 		return p
 	} else {
 		p := &noStreamProcessor{}
 		p.opt = opt
+		p.hasDoc = spec.HasDoc
 		return p
 	}
 }
@@ -91,6 +95,8 @@ type options struct {
 type processor struct {
 	processors.Base
 
+	hasDoc bool
+
 	opt *options
 
 	flagsTpl *template.Template
@@ -268,6 +274,65 @@ func (p *processor) buildCommandArgs(e processors.IPacket) []string {
 
 	return finalArgs
 }
+func (p *processor) Doc() *doc.Processor {
+	d := &doc.Processor{
+		Behavior: p.opt.Behavior,
+		Name:     p.Name,
+		Doc:      "",
+		DocShort: "",
+		Options: &doc.ProcessorOptions{
+			Options: []*doc.ProcessorOption{},
+		},
+	}
+
+	if p.hasDoc == false {
+		return d
+	}
+
+	args := p.buildCommandArgs(nil)
+	args = append(args, "--help-json")
+	out, err := exec.Command(p.opt.Command, args...).Output()
+	if err != nil {
+		return d
+	}
+
+	var dat map[string]interface{}
+	if err := json.Unmarshal(out, &dat); err != nil {
+		return d
+	}
+
+	d.Doc = dat["Description"].(string)
+	d.DocShort = dat["ShortDescription"].(string)
+
+	for _, opti := range dat["Options"].(map[string]interface{}) {
+		opt := opti.(map[string]interface{})
+
+		d.Options.Options = append(d.Options.Options, &doc.ProcessorOption{
+			Name:         opt["name"].(string),
+			Alias:        opt["name"].(string),
+			Doc:          opt["doc"].(string),
+			Required:     opt["required"].(bool),
+			DefaultValue: opt["default_value"],
+			Type:         mapType(opt["type"].(string)),
+			// PossibleValues: []string{},
+			// ExampleLS:      "",
+		})
+	}
+
+	return d
+}
+
+func mapType(s string) string {
+	switch s {
+	case "[]string":
+		return "array"
+	case "[]int":
+		return "array"
+	case "map[string]string":
+		return "hash"
+	}
+	return s
+}
 
 func (p *processor) startCommand(e processors.IPacket) (*exec.Cmd, io.WriteCloser, io.ReadCloser, io.ReadCloser, error) {
 	args := p.buildCommandArgs(e)

store/xprocessors.go

@@ -25,6 +25,7 @@ type StoreXProcessor struct {
 	StdinAs               string   `json:"stdin_as"`
 	StdoutAs              string   `json:"stdout_as"`
 	OptionsCompositionTpl string   `json:"options_composition_tpl"`
+	HasDoc                bool     `json:"has_doc"`
 }
 
 func (s *Store) CreateXProcessor(xp *models.XProcessor) {
@@ -44,6 +45,7 @@ func (s *Store) CreateXProcessor(xp *models.XProcessor) {
 		StdoutAs:              xp.StdoutAs,
 		Description:           xp.Description,
 		OptionsCompositionTpl: xp.OptionsCompositionTpl,
+		HasDoc:                xp.HasDoc,
 	}
 
 	s.db.Upsert(sxp.Uuid, sxp)
@@ -66,6 +68,7 @@ func (s *Store) SaveXProcessor(xp *models.XProcessor) {
 		StdoutAs:              xp.StdoutAs,
 		Description:           xp.Description,
 		OptionsCompositionTpl: xp.OptionsCompositionTpl,
+		HasDoc:                xp.HasDoc,
 	}
 
 	s.db.Upsert(sp.Uuid, sp)
@@ -106,6 +109,7 @@ func (s *Store) FindXProcessors(behavior string) []models.XProcessor {
 		tXProcessor.StdoutAs = xp.StdoutAs
 		tXProcessor.Description = xp.Description
 		tXProcessor.OptionsCompositionTpl = xp.OptionsCompositionTpl
+		tXProcessor.HasDoc = xp.HasDoc
 
 		xps = append(xps, tXProcessor)
 	}
@@ -139,6 +143,7 @@ func (s *Store) FindOneXProcessorByUUID(UUID string, withAssetValues bool) (mode
 	tXProcessor.StdoutAs = sps[0].StdoutAs
 	tXProcessor.Description = sps[0].Description
 	tXProcessor.OptionsCompositionTpl = sps[0].OptionsCompositionTpl
+	tXProcessor.HasDoc = sps[0].HasDoc
 
 	return tXProcessor, nil
 }
@@ -170,6 +175,7 @@ func (s *Store) FindOneXProcessorByName(name string) (models.XProcessor, error)
 	tXProcessor.StdoutAs = sps[0].StdoutAs
 	tXProcessor.Description = sps[0].Description
 	tXProcessor.OptionsCompositionTpl = sps[0].OptionsCompositionTpl
+	tXProcessor.HasDoc = sps[0].HasDoc
 
 	return tXProcessor, nil
 }