From 1602507d308cf163959505241cf1f7d533a49528 Mon Sep 17 00:00:00 2001 From: Craig Johnston Date: Mon, 13 May 2019 17:55:31 -0700 Subject: [PATCH] templated queries --- README.md | 30 ++++++++++ cmd/query.go | 12 ++-- docker-compose.yml | 2 +- go.mod | 38 +++++++++++-- go.sum | 53 ++++++++++++++++++ query.go | 135 +++++++++++++++++++++++++++++++++++++++------ 6 files changed, 239 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index a56025c..59ffc68 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,36 @@ curl -X GET \ -H 'Authorization: Basic dGVzdDpQRFdnWXIzYlFHTm9McHRCUkRrTFRHUWNSbUNNcUxHUkZwWG9YSjh4TVBzTUxNZzNMSHZXcEpnRHUydjNMWUJB' ``` +## Query Templates + +Query templates use [Go's build in template](https://golang.org/pkg/text/template/) system along with [Sprig](http://masterminds.github.io/sprig/). The following example tests a query with `/run` and +demonstrates a query string override of default values provided in the `"parameters"` section: + +```bash +curl -X POST \ + 'http://localhost:8080/run/test?size=0&idx=2019.%2A' \ + -H 'Authorization: Basic dGVzdDpQRFdnWXIzYlFHTm9McHRCUkRrTFRHUWNSbUNNcUxHUkZwWG9YSjh4TVBzTUxNZzNMSHZXcEpnRHUydjNMWUJB' \ + -d '{ + "machine_name": "get_some_metrics", + "display_name": "Get records from the some_metrics index.", + "description": "Get some_metrics", + "model": "some_metrics", + "idx_pattern": "-ts-{{ index . \"idx\" }}", + "query_template": "{\"size\": {{ index . \"size\" }},\"query\": {\"match_all\": {}}}", + "parameters": [ + { + "machine_name": "size", + "default_value": "1" + }, + { + "machine_name": "idx", + "default_value": "2019.*" + } + ] +}' +``` + + [Token]: https://github.com/txn2/token [txn2/provision]: https://github.com/txn2/provision [txn2/tm]: https://github.com/txn2/tm diff --git a/cmd/query.go b/cmd/query.go index bdc68f0..87b4ee5 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -13,6 +13,7 @@ package main import ( "bytes" + "encoding/json" "flag" "fmt" "net/http" @@ -21,7 +22,6 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/gin-gonic/gin/json" "github.com/patrickmn/go-cache" "github.com/txn2/ack" "github.com/txn2/micro" @@ -83,7 +83,6 @@ func main() { ak := ack.Gin(c) ak.SetPayload("Unauthorized via cache.") ak.GinErrorAbort(401, "E401", "UnauthorizedFailure") - return } @@ -124,16 +123,17 @@ func main() { tokenHandler := provision.UserTokenHandler() tokenHandler(c) + if c.IsAborted() { + return + } + // Check token tokenCheck := provision.AccountAccessCheckHandler(checkAdmin) tokenCheck(c) } } - // User token middleware - //server.Router.Use() - - // run a query (one-off operation for running or testing queries" + // Run a query (one-off operation for running or testing queries server.Router.POST("run/:account", accessHandler(false), qApi.RunQueryHandler, diff --git a/docker-compose.yml b/docker-compose.yml index 5b0b74c..0fdadee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,7 +100,7 @@ services: # the properties of the rxtx payload. # see https://github.com/txn2/tm tm: - image: txn2/tm:v0.1.1 + image: txn2/tm:v0.2.0 container_name: tm restart: unless-stopped environment: diff --git a/go.mod b/go.mod index 95c9189..ed7b1e7 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,38 @@ module github.com/txn2/query go 1.12 require ( + github.com/Masterminds/goutils v1.1.0 // indirect + github.com/Masterminds/semver v1.4.2 // indirect + github.com/Masterminds/sprig v2.18.0+incompatible + github.com/beorn7/perks v1.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 github.com/gin-gonic/gin v1.3.0 + github.com/go-logfmt/logfmt v0.4.0 // indirect + github.com/gogo/protobuf v1.2.1 // indirect + github.com/google/uuid v1.1.1 // indirect + github.com/huandu/xstrings v1.2.0 // indirect + github.com/imdario/mergo v0.3.7 // indirect + github.com/json-iterator/go v1.1.6 // indirect + github.com/kisielk/errcheck v1.2.0 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect + github.com/kr/pty v1.1.4 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect - github.com/txn2/ack v1.8.2 - github.com/txn2/es v1.3.5 - github.com/txn2/micro v0.0.5 - github.com/txn2/provision v0.0.8 - github.com/txn2/tm v0.0.10 - go.uber.org/zap v1.9.1 + github.com/prometheus/common v0.4.0 // indirect + github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 // indirect + github.com/sirupsen/logrus v1.4.1 // indirect + github.com/stretchr/objx v0.2.0 // indirect + github.com/txn2/ack v1.8.3 + github.com/txn2/es v1.3.6 + github.com/txn2/micro v0.0.6 + github.com/txn2/provision v0.1.4 + github.com/txn2/tm v0.2.3 + github.com/ugorji/go v1.1.2-0.20180831062425-e253f1f20942 // indirect + go.uber.org/atomic v1.4.0 // indirect + go.uber.org/zap v1.10.0 + golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f // indirect + golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5 // indirect + golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 // indirect + golang.org/x/text v0.3.2 // indirect + golang.org/x/tools v0.0.0-20190513184735-d81a07b7e584 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index c756157..f6e42db 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,15 @@ +github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg= +github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/sprig v2.18.0+incompatible h1:QoGhlbC6pter1jxKnjMFxT8EqsLuDE6FEcNbWEpw+lI= +github.com/Masterminds/sprig v2.18.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -14,6 +22,8 @@ github.com/gin-contrib/zap v0.0.0-20190405225521-7c4b822813e7 h1:v6rNWmMnVDBVMc1 github.com/gin-contrib/zap v0.0.0-20190405225521-7c4b822813e7/go.mod h1:pQKeeey3PeRN2SbZe1jWiIkTJkylO9hL1K0Hf4Wbtt4= github.com/gin-gonic/gin v1.3.0 h1:kCmZyPklC0gVdL728E6Aj20uYBJV93nj/TkwBTKhFbs= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= +github.com/gin-gonic/gin v1.4.0 h1:3tMoCCfM7ppqsR0ptz/wi1impNpT7/9wQtMZ8lr1mCQ= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -25,6 +35,12 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= +github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= +github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= +github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -38,6 +54,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -69,11 +86,15 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jO github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.3.0 h1:taZ4h8Tkxv2kNyoSctBvfXEHmBmxrwmIidZTIaHons4= github.com/prometheus/common v0.3.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142 h1:JO6VBMEDSBX/LT4GKwSdvuFigZNwVD4lkPyUE4BDCKE= github.com/prometheus/procfs v0.0.0-20190412120340-e22ddced7142/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -84,39 +105,67 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/txn2/ack v1.8.2 h1:3k47TVg9hVuGDHkhfCGBKqimBawR7QCwvPoRdpQhQ58= github.com/txn2/ack v1.8.2/go.mod h1:tP7ACAXVBDjmPPtl1WO1sd1VkKks69HUQsQYPf7NXKw= +github.com/txn2/ack v1.8.3 h1:Jie4nVJX/kT4APF0ZVr6lbwlJyR0lAA47e3nRYzr19U= +github.com/txn2/ack v1.8.3/go.mod h1:tP7ACAXVBDjmPPtl1WO1sd1VkKks69HUQsQYPf7NXKw= github.com/txn2/es v1.3.3 h1:xtv2RSaGTr6yF2dhfCj72J3IULYfFRqtk9P9bGXanuA= github.com/txn2/es v1.3.3/go.mod h1:EpfYehD9ztuADfvJo9Eoiv43yampBl/F5MaNNb6C7WU= github.com/txn2/es v1.3.5 h1:BLlRQ4DnhwjEKYHoI+xsvaSV4En+h/gzA+knWgYi7Og= github.com/txn2/es v1.3.5/go.mod h1:EpfYehD9ztuADfvJo9Eoiv43yampBl/F5MaNNb6C7WU= +github.com/txn2/es v1.3.6 h1:iQGO3IOapPWtXIakuBhJ6/9Neb+j5bdGinbxRY0/2sM= +github.com/txn2/es v1.3.6/go.mod h1:EpfYehD9ztuADfvJo9Eoiv43yampBl/F5MaNNb6C7WU= github.com/txn2/micro v0.0.5 h1:w6TcgFnYfbBW2r+FUM7MIUTN4q7+y/GvGqewquTt9tk= github.com/txn2/micro v0.0.5/go.mod h1:r/uIbdFhvfjqKcT8mQUIKmzuh2Rxcablu7KB8oRRBoo= +github.com/txn2/micro v0.0.6 h1:VbN+fCcmRcagVhXYSlUCYgftkTTb5AZO1IltIJ/ZNfA= +github.com/txn2/micro v0.0.6/go.mod h1:r/uIbdFhvfjqKcT8mQUIKmzuh2Rxcablu7KB8oRRBoo= github.com/txn2/provision v0.0.8 h1:PFcSIShdPq87FOMnOVvAlMiXHhlcCTUH5nB/vg+7OtM= github.com/txn2/provision v0.0.8/go.mod h1:4tzJEpK1LLgASok2bNvhZlXw1h98aP6+GOvxj5gXifY= +github.com/txn2/provision v0.1.4 h1:4WtDgZnxofhcnA1Sa9rkDUtajWDp8Of3jBKBR/KhgGo= +github.com/txn2/provision v0.1.4/go.mod h1:+8d+mi6SAHRYfTZxLMf0043gUclxnFyozoH0vktEFko= github.com/txn2/tm v0.0.10 h1:rNXZZr6zHPCLZHafmg4EI40LMgn7tMQpEdTdE4g8rTM= github.com/txn2/tm v0.0.10/go.mod h1:mcGxYehiZu8xuUFxsdHEu4ZSNNaKk8Q9XUTBsFVEHnI= +github.com/txn2/tm v0.2.0 h1:an4NkNVcpCfmFAUsY6ghnfaivrpCqpX/ZsRJ3uw/dL0= +github.com/txn2/tm v0.2.0/go.mod h1:mcGxYehiZu8xuUFxsdHEu4ZSNNaKk8Q9XUTBsFVEHnI= +github.com/txn2/tm v0.2.1 h1:jQzssurCNTsl+esG93N+C7B1iGbynRDBTnoCZrynVQg= +github.com/txn2/tm v0.2.1/go.mod h1:mcGxYehiZu8xuUFxsdHEu4ZSNNaKk8Q9XUTBsFVEHnI= +github.com/txn2/tm v0.2.3 h1:t/BnNFHVx/iVJhJfG7MP4J68GGdJHoZg/tT8VK8fpf4= +github.com/txn2/tm v0.2.3/go.mod h1:mcGxYehiZu8xuUFxsdHEu4ZSNNaKk8Q9XUTBsFVEHnI= github.com/txn2/token v0.0.1 h1:4nAK3duT59mXuZgvAi4Dmt6WNaOa17s+e6KiOBmIgHE= github.com/txn2/token v0.0.1/go.mod h1:rwlndPTGA4xxAuDI/cG2Tv43PJqDWFbFv7Vqm7nQbtc= +github.com/ugorji/go v1.1.2-0.20180831062425-e253f1f20942 h1:C/c80f5WTivlgVJaxjYkGosvyNLVlehHXvTwubC5AnY= +github.com/ugorji/go v1.1.2-0.20180831062425-e253f1f20942/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2 h1:EICbibRW4JNKMcY+LsWmuwob+CRS1BmdRdjphAm9mH4= github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a h1:Igim7XhdOpBnWPuYJ70XcNpq8q3BCACtVgNfoJxOV7g= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -125,10 +174,14 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190411180116-681f9ce8ac52/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190513184735-d81a07b7e584/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/query.go b/query.go index 562c611..ffb6d03 100644 --- a/query.go +++ b/query.go @@ -14,9 +14,14 @@ package query import ( + "bytes" "encoding/json" "fmt" "strings" + "text/template" + "time" + + "github.com/Masterminds/sprig" "github.com/gin-gonic/gin" "github.com/txn2/ack" @@ -57,6 +62,23 @@ func NewApi(cfg *Config) (*Api, error) { }) } + // check for elasticsearch a few times before failing + // this reduces a reliance on restarts when a full system is + // spinning up + backOff := []int{10, 10, 15, 15, 30, 30, 45} + for _, boff := range backOff { + code, _, _ := a.Elastic.Get("") + a.Logger.Info("Attempting to contact Elasticsearch", zap.String("server", a.Elastic.ElasticServer)) + + if code == 200 { + a.Logger.Info("Connection to Elastic search successful.", zap.String("server", a.Elastic.ElasticServer)) + break + } + + a.Logger.Warn("Unable to contact Elasticsearch rolling back off.", zap.Int("wait_seconds", boff)) + <-time.After(time.Duration(boff) * time.Second) + } + // send template mappings for query index _, _, err := a.Elastic.SendEsMapping(GetQueryTemplateMapping()) if err != nil { @@ -81,7 +103,7 @@ func (a *Api) RunQueryHandler(c *gin.Context) { return } - code, queryExecuteResult, err := a.ExecuteQuery(account, *query) + code, queryExecuteResult, err := a.ExecuteQuery(account, *query, c) if err != nil { a.Logger.Error("EsError", zap.Error(err)) ak.SetPayloadType("EsError") @@ -101,9 +123,68 @@ func (a *Api) RunQueryHandler(c *gin.Context) { } // ExecuteQuery -func (a *Api) ExecuteQuery(account string, query Query) (int, *es.Obj, error) { +func (a *Api) ExecuteQuery(account string, query Query, c *gin.Context) (int, *es.Obj, error) { queryResults := &es.Obj{} + // is there a template to process? + // + if query.QueryTemplate != "" { + // populate parameter map + params := make(map[string]interface{}) + for _, param := range query.Parameters { + if qv, ok := c.GetQuery(param.MachineName); ok { + params[param.MachineName] = qv + continue + } + params[param.MachineName] = param.DefaultValue + } + + // process template + a.Logger.Debug("Process query template.") + tmpl, err := template.New("query_template").Funcs(sprig.TxtFuncMap()).Parse(query.QueryTemplate) + if err != nil { + a.Logger.Error("Error processing query template.", zap.Error(err)) + return 500, nil, err + } + + var qb bytes.Buffer + err = tmpl.Execute(&qb, params) + if err != nil { + a.Logger.Error("Error executing query template.", zap.Error(err)) + return 500, nil, err + } + + query.QueryJson = qb.String() + a.Logger.Debug("Parsed template query", zap.String("query", query.QueryJson)) + + query.Query = &es.Obj{} + + err = json.Unmarshal(qb.Bytes(), query.Query) + if err != nil { + a.Logger.Error("Error Marshaling query json into object.", zap.Error(err)) + return 500, nil, err + } + + // process index pattern as template + // + a.Logger.Debug("Process idx_pattern template.") + tmpl, err = template.New("idx_pattern").Funcs(sprig.TxtFuncMap()).Parse(query.IdxPattern) + if err != nil { + a.Logger.Error("Error processing idx_pattern template.", zap.Error(err)) + return 500, nil, err + } + + var ipb bytes.Buffer + err = tmpl.Execute(&ipb, params) + if err != nil { + a.Logger.Error("Error executing idx_pattern template.", zap.Error(err)) + return 500, nil, err + } + + query.IdxPattern = ipb.String() + + } + path := fmt.Sprintf("%s-data-%s%s/_search", account, query.Model, query.IdxPattern) code, err := a.Elastic.PostObjUnmarshal(path, query.Query, queryResults) @@ -140,17 +221,20 @@ func (a *Api) ExecuteQueryHandler(c *gin.Context) { return } - qObj := &es.Obj{} + // only attempt if there is no template + if queryResult.Source.QueryTemplate == "" { + qObj := &es.Obj{} - err = json.Unmarshal([]byte(queryResult.Source.QueryJson), qObj) - if err != nil { - ak.GinErrorAbort(500, "QueryUnmarshalError", err.Error()) - return - } + err = json.Unmarshal([]byte(queryResult.Source.QueryJson), qObj) + if err != nil { + ak.GinErrorAbort(500, "QueryUnmarshalError", err.Error()) + return + } - queryResult.Source.Query = qObj + queryResult.Source.Query = qObj + } - code, queryExecuteResult, err := a.ExecuteQuery(account, queryResult.Source) + code, queryExecuteResult, err := a.ExecuteQuery(account, queryResult.Source, c) if err != nil { a.Logger.Error("EsError", zap.Error(err)) ak.SetPayloadType("EsError") @@ -191,15 +275,17 @@ func (a *Api) UpsertQueryHandler(c *gin.Context) { return } - // convert query to json - queryJson, err := json.Marshal(query.Query) - if err != nil { - ak.GinErrorAbort(500, "QueryUnmashalError", err.Error()) - return - } + if query.Query != nil { + // convert query to json + queryJson, err := json.Marshal(query.Query) + if err != nil { + ak.GinErrorAbort(500, "QueryUnmashalError", err.Error()) + return + } - query.QueryJson = string(queryJson) - query.Query = nil + query.QueryJson = string(queryJson) + query.Query = nil + } // ensure lowercase machine name query.MachineName = strings.ToLower(query.MachineName) @@ -319,9 +405,16 @@ type Query struct { // query object Query *es.Obj `json:"query,omitempty" mapstructure:"query"` + // used to describe input parameters + Parameters []tm.Model `json:"parameters,omitempty" mapstructure:"parameters"` + // query json QueryJson string `json:"query_json,omitempty" mapstructure:"query_json"` + // if a query template is present it will take the place of + // the query and query_json fields + QueryTemplate string `json:"query_template,omitempty" mapstructure:"query_template"` + // describes the query output ResultFields []tm.Model `json:"fields" mapstructure:"fields"` } @@ -359,6 +452,12 @@ func GetQueryTemplateMapping() es.IndexTemplate { "query_json": es.Obj{ "type": "text", }, + "query_template": es.Obj{ + "type": "text", + }, + "parameters": es.Obj{ + "type": "nested", + }, "result_fields": es.Obj{ "type": "nested", },