Refactor bitfan binaries (#47)

* move cmd/ to cli/

* bitfanDoc is now in cmd/bitfanDoc

* bitfanUI now in cmd/bitfanUI

* fix bitfanUI import

* ignore dist, add vendoring assetfs

* update readme

* api : configuration was empty when pipeline start
ui : fix import path
ui : template were not reload on each request on dev mode (--dev)
core : storage - FindOnePipelineByUUID can now return a pipeline with asset's values when needed

* move main.go bitfan to cmd/bitfan

* goreleaser : update binaries main

* bitfan : main - fix go generate

* remove .bitfan dir

* ignore

* processor : grok - fix multi match with break_on_match

* bitfan - fix legacy logstash style flags (deprecated flags)

* ignore notes

* bitfan cli : when no know command is provided then use the "run" one.
	`bitfan myconfig.conf` accepted and runned as `bitfan run myconfig.conf`

.gitignore

@@ -25,7 +25,7 @@ _testmain.go
 
 tester
 examples.d/twitter.conf
-bitfan
+
 releases
 .DS_Store
 
@@ -56,3 +56,9 @@ releases
 /ui/cmd/test.db
 
 /webui/webui
+
+/dist/
+
+/cmd/bitfan/.bitfan/
+
+/cmd/bitfan/notes.md

.goreleaser.yml

@@ -1,7 +1,19 @@
 # .goreleaser.yml
 # Build customization
 builds:
-  -
+  - binary: bitfan
+    main: ./cmd/bitfan/main.go
+    goos:
+      - windows
+      - darwin
+      - linux
+    goarch:
+      - amd64
+      - arm
+    goarm:
+      - 7
+  - binary: bitfanUI
+    main: ./cmd/bitfanUI/main.go
     goos:
       - windows
       - darwin

README.md

@@ -39,7 +39,7 @@
 - [x] manage running pipelines (list / stop / start a pipeline in a running bitfan)
 - [x] monitor pipeline processors and events with prometheus
 - [x] REST API to manage Bitfan
-- [x] WebUI : https://github.com/vjeantet/bitfan-ui
+- [x] WebUI
 
 
 

api/handler.go

@@ -13,11 +13,11 @@ func init() {
 	gin.SetMode(gin.ReleaseMode)
 }
 
-var logger *core.Logger
+var apiLogger *core.Logger
 
 func Handler(path string) http.Handler {
 
-	logger = core.NewLogger("api", nil)
+	apiLogger = core.NewLogger("api", nil)
 
 	logs, _ := newHook(hookConfig{Size: 100})
 	logrus.AddHook(logs)
@@ -79,7 +79,7 @@ func Handler(path string) http.Handler {
 		// v1.GET("/docs/outputs/:name", getDocsOutputsByName)
 	}
 
-	logger.Debugf("Serving API on /%s/ ", path)
+	apiLogger.Debugf("Serving API on /%s/ ", path)
 
 	return r
 }

api/pipelines.go

@@ -62,7 +62,7 @@ func (p *PipelineApiController) Find(c *gin.Context) {
 
 func (p *PipelineApiController) FindOneByUUID(c *gin.Context) {
 	uuid := c.Param("uuid")
-	mPipeline, err := core.Storage().FindOnePipelineByUUID(uuid)
+	mPipeline, err := core.Storage().FindOnePipelineByUUID(uuid, false)
 	if err != nil {
 		c.JSON(404, models.Error{Message: err.Error()})
 		return
@@ -81,7 +81,7 @@ func (p *PipelineApiController) FindOneByUUID(c *gin.Context) {
 func (p *PipelineApiController) UpdateByUUID(c *gin.Context) {
 	uuid := c.Param("uuid")
 
-	mPipeline, err := core.Storage().FindOnePipelineByUUID(uuid)
+	mPipeline, err := core.Storage().FindOnePipelineByUUID(uuid, false)
 	if err != nil {
 		c.JSON(404, models.Error{Message: err.Error()})
 		return
@@ -109,7 +109,7 @@ func (p *PipelineApiController) UpdateByUUID(c *gin.Context) {
 		case true: // pipeline is on
 			switch nextActive {
 			case true: // restart
-				core.Log().Debugf("restarting pipeline %s", uuid)
+				apiLogger.Debugf("restarting pipeline %s", uuid)
 				err := core.StopPipeline(uuid)
 				if err != nil {
 					c.JSON(500, models.Error{Message: err.Error()})
@@ -121,7 +121,7 @@ func (p *PipelineApiController) UpdateByUUID(c *gin.Context) {
 					return
 				}
 			case false: // stop
-				core.Log().Debugf("stopping pipeline %s", uuid)
+				apiLogger.Debugf("stopping pipeline %s", uuid)
 				err := core.StopPipeline(uuid)
 				if err != nil {
 					c.JSON(500, models.Error{Message: err.Error()})
@@ -131,7 +131,7 @@ func (p *PipelineApiController) UpdateByUUID(c *gin.Context) {
 		case false: // pipeline is off
 			switch nextActive {
 			case true: // start pipeline
-				core.Log().Debugf("starting pipeline %s", uuid)
+				apiLogger.Debugf("starting pipeline %s", uuid)
 				err := core.StartPipelineByUUID(uuid)
 				if err != nil {
 					c.JSON(500, models.Error{Message: err.Error()})
@@ -151,7 +151,7 @@ func (p *PipelineApiController) UpdateByUUID(c *gin.Context) {
 func (p *PipelineApiController) DeleteByUUID(c *gin.Context) {
 	uuid := c.Param("uuid")
 
-	mPipeline, err := core.Storage().FindOnePipelineByUUID(uuid)
+	mPipeline, err := core.Storage().FindOnePipelineByUUID(uuid, false)
 	if err != nil {
 		c.JSON(404, models.Error{Message: err.Error()})
 		return

cmd/cmd_test.go → cmd/bitfan/commands/cmd_test.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import "testing"
 

cmd/conf.go → cmd/bitfan/commands/conf.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import (
 	"fmt"

cmd/doc.go → cmd/bitfan/commands/doc.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package cmd
+package commands
 
 import (
 	"fmt"

cmd/list.go → cmd/bitfan/commands/list.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package cmd
+package commands
 
 import (
 	"fmt"

cmd/plugins.go → cmd/bitfan/commands/plugins.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import (
 	"github.com/vjeantet/bitfan/core"

cmd/root.go → cmd/bitfan/commands/root.go

@@ -12,12 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package cmd
+package commands
 
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"runtime"
+	"strings"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -34,6 +36,8 @@ var RootCmd = &cobra.Command{
 		viper.BindPFlag("log", cmd.Flags().Lookup("log"))
 		viper.BindPFlag("verbose", cmd.Flags().Lookup("verbose"))
 		viper.BindPFlag("debug", cmd.Flags().Lookup("debug"))
+		viper.BindPFlag("data", cmd.Flags().Lookup("data"))
+		viper.BindPFlag("host", cmd.Flags().Lookup("host"))
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 
@@ -68,6 +72,14 @@ var RootCmd = &cobra.Command{
 // Execute adds all child commands to the root command sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
+	// Workarround to run as "bitfan run" if no know command where found
+	// waiting https://github.com/spf13/cobra/pull/369 to delete this
+	_, _, err := RootCmd.Find(os.Args[1:])
+	if err != nil && strings.HasPrefix(err.Error(), "unknown command") {
+		// insert "run" in args
+		os.Args = append(os.Args[:1], append([]string{"run"}, os.Args[1:]...)...)
+	}
+
 	if err := RootCmd.Execute(); err != nil {
 		fmt.Println("cmd execute error: ", err)
 		os.Exit(-1)
@@ -105,12 +117,17 @@ func init() {
 	RootCmd.Flags().StringP("eval", "e", "", "Use the given string as the configuration data.")
 	RootCmd.Flags().BoolP("version", "V", false, "Display version info.")
 	RootCmd.Flags().IntP("workers", "w", runtime.NumCPU(), "number of workers")
+	cwd, _ := os.Getwd()
+	RootCmd.Flags().String("data", filepath.Join(cwd, ".bitfan"), "Path to data dir")
+	RootCmd.Flags().StringP("host", "H", "127.0.0.1:5123", "Service Host to connect to")
 
 	RootCmd.Flags().MarkDeprecated("config", "use the run command")
 	RootCmd.Flags().MarkDeprecated("configtest", "use the test command")
 	RootCmd.Flags().MarkDeprecated("eval", "use the run or test command")
 	RootCmd.Flags().MarkDeprecated("version", "use the version command")
 	RootCmd.Flags().MarkHidden("workers")
+	RootCmd.Flags().MarkHidden("data")
+	RootCmd.Flags().MarkHidden("host")
 
 	RootCmd.PersistentFlags().String("settings", "current dir, then ~/.bitfan/ then /etc/bitfan/", "Set the directory containing the bitfan.toml settings")
 	RootCmd.PersistentFlags().StringP("log", "l", "", "Log to a given path. Default is to log to stdout.")

cmd/run.go → cmd/bitfan/commands/run.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package cmd
+package commands
 
 import (
 	"os"

cmd/service.go → cmd/bitfan/commands/service.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import (
 	"log"

cmd/serviceInstall.go → cmd/bitfan/commands/serviceInstall.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import (
 	"fmt"

cmd/serviceRemove.go → cmd/bitfan/commands/serviceRemove.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import (
 	"log"

cmd/serviceRestart.go → cmd/bitfan/commands/serviceRestart.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import (
 	"log"

cmd/serviceStart.go → cmd/bitfan/commands/serviceStart.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import (
 	"log"

cmd/serviceStop.go → cmd/bitfan/commands/serviceStop.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import (
 	"log"

cmd/start.go → cmd/bitfan/commands/start.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import (
 	"fmt"

cmd/stop.go → cmd/bitfan/commands/stop.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import (
 	"fmt"

cmd/test.go → cmd/bitfan/commands/test.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import (
 	"fmt"

cmd/version.go → cmd/bitfan/commands/version.go

@@ -1,4 +1,4 @@
-package cmd
+package commands
 
 import (
 	"fmt"

main.go → cmd/bitfan/main.go

@@ -1,6 +1,6 @@
 //go:generate go generate github.com/vjeantet/bitfan/processors/...
 //go:generate go generate github.com/vjeantet/bitfan/codecs/...
-//go:generate swagger generate spec -m -b github.com/vjeantet/bitfan/api -o api/swagger.json
+//go:generate swagger generate spec -m -b github.com/vjeantet/bitfan/api -o ../../api/swagger.json
 // Copyright © 2016 Valere JEANTET <[email protected]>
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -23,10 +23,10 @@ import (
 	"runtime"
 
 	"github.com/kardianos/service"
-	"github.com/vjeantet/bitfan/cmd"
+	"github.com/vjeantet/bitfan/cmd/bitfan/commands"
 )
 
-var version = "dev"
+var version = "master"
 var buildstamp = ""
 
 func main() {
@@ -37,7 +37,7 @@ func main() {
 	if !service.Interactive() {
 
 		// PASS Service
-		s := cmd.GetService()
+		s := commands.GetService()
 
 		slogger, err := s.Logger(nil)
 		if err != nil {
@@ -52,8 +52,8 @@ func main() {
 	}
 
 	//interactive
-	cmd.Version = version
-	cmd.Buildstamp = buildstamp
-	cmd.Execute()
+	commands.Version = version
+	commands.Buildstamp = buildstamp
+	commands.Execute()
 
 }