From 1daf6047899e7804fd30cc950eddda094af0a3a4 Mon Sep 17 00:00:00 2001 From: Boris Date: Mon, 14 Oct 2024 13:53:17 +0300 Subject: [PATCH] =?UTF-8?q?=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D0=B0=20=D1=81?= =?UTF-8?q?=20index/mappings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- front/search.html | 51 ++++++++++++++++++---- go.mod | 1 + modules/router/methods.go | 19 ++++++++ modules/router/router.go | 91 ++++++++++++++++++++++++++------------- 4 files changed, 123 insertions(+), 39 deletions(-) diff --git a/front/search.html b/front/search.html index 544b0e0..03ac578 100644 --- a/front/search.html +++ b/front/search.html @@ -65,7 +65,7 @@ - + @@ -103,7 +103,6 @@ - @@ -112,6 +111,7 @@ const date = new Date(); +var mapping = {} date.setMinutes(date.getMinutes() - 15) $.datetimepicker.setLocale('ru'); $('#datetimepicker_start').datetimepicker({timepicker: true, format:'Y-m-d H:i:s', step: 15, value:date}); @@ -207,7 +207,7 @@ "; + str += "
  •  "+ k + "(" + data[k] + ")" +"
    • "; } $("#fields").html(str); $("#loading").addClass('invisible'); @@ -218,11 +218,13 @@ X-tract indices from snapshot"; + + if ( res.length > 0 ) { + str = ""; + if ( fields.length == 0 ) { + str+=""; + } else { + for (var f in res[0].fields) { + if (f==tf[0]) { + continue; + } else { + str+=""; + } + + } + } + str+=""; + for (var k in res) { + str += ""; + if ( fields.length == 0 ) { + str += ""; + str += ""; + } else { + str += ""; + for (var f in res[k].fields) { + if (f==tf[0]) { + continue; + } else { + str+=""; + } + } + } + str+=""; + } + str+="
    Time_source" + f + "
    "+ res[k]._source[tf[0]] +""+ JSON.stringify(res[k]._source); +""+ res[k].fields[tf[0]] +"" + res[k].fields[f]+"
    "; + } else { + str = "

    No search results found

    "; } $("#result").html(str); }, diff --git a/go.mod b/go.mod index 55e5532..d41571a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/jteeuwen/go-bindata v3.0.7+incompatible // indirect github.com/uzhinskiy/lib.go v0.1.7 + github.com/Jeffail/gabs/v2 v2.7.0 gopkg.in/yaml.v2 v2.3.0 github.com/opensearch-project/opensearch-go v1.1.0 ) diff --git a/modules/router/methods.go b/modules/router/methods.go index d5fa1c4..c60ca12 100644 --- a/modules/router/methods.go +++ b/modules/router/methods.go @@ -233,3 +233,22 @@ func (rt *Router) Barrel(ind_array IndicesInSnap, s3 bool) ([]string, []string) } return a, b } + +func (rt *Router) flattenMap(prefix string, nestedMap map[string]interface{}, flatMap map[string]string) { + for key, value := range nestedMap { + fullKey := key + if prefix != "" { + fullKey = prefix + "." + key + } + + if subMap, ok := value.(map[string]interface{}); ok { + if typeVal, exists := subMap["type"]; exists { + flatMap[fullKey] = typeVal.(string) + } + + if props, hasProps := subMap["properties"]; hasProps { + rt.flattenMap(fullKey, props.(map[string]interface{}), flatMap) + } + } + } +} diff --git a/modules/router/router.go b/modules/router/router.go index 5e6ece3..31a0a5a 100644 --- a/modules/router/router.go +++ b/modules/router/router.go @@ -27,6 +27,7 @@ import ( "time" + //"github.com/Jeffail/gabs/v2" "github.com/flant/elasticsearch-extractor/modules/config" "github.com/flant/elasticsearch-extractor/modules/front" "github.com/flant/elasticsearch-extractor/modules/version" @@ -43,17 +44,18 @@ type Router struct { type apiRequest struct { Action string `json:"action,omitempty"` // Имя вызываемого метода* Values struct { - Indices []string `json:"indices,omitempty"` - Repo string `json:"repo,omitempty"` - OrderDir string `json:"odir,omitempty"` - OrderType string `json:"otype,omitempty"` - Snapshot string `json:"snapshot,omitempty"` - Index string `json:"index,omitempty"` - Cluster string `json:"cluster,omitempty"` - Xql string `json:"xql,omitempty"` - Fields []string `json:"fields,omitempty"` - DateStart string `json:"date_start,omitempty"` - DateEnd string `json:"date_end,omitempty"` + Indices []string `json:"indices,omitempty"` + Repo string `json:"repo,omitempty"` + OrderDir string `json:"odir,omitempty"` + OrderType string `json:"otype,omitempty"` + Snapshot string `json:"snapshot,omitempty"` + Index string `json:"index,omitempty"` + Cluster string `json:"cluster,omitempty"` + Xql string `json:"xql,omitempty"` + Fields []string `json:"fields,omitempty"` + Timefields []string `json:"timefields,omitempty"` + DateStart string `json:"date_start,omitempty"` + DateEnd string `json:"date_end,omitempty"` } `json:"values,omitempty"` } @@ -113,14 +115,6 @@ type indexGroup struct { Index string `json:"index,omitempty"` } -type IndexMapping map[string]struct { - Mappings struct { - Properties map[string]struct { - Type string - } - } -} - type Cluster struct { Name string Host string @@ -518,25 +512,35 @@ func (rt *Router) ApiHandler(w http.ResponseWriter, r *http.Request) { case "get_mapping": { t := time.Now() - var m IndexMapping + var ( + fullm map[string]interface{} + m map[string]interface{} + ) + + flatMap := make(map[string]string) response, err := rt.doGet(request.Values.Cluster + request.Values.Index + t.Format("2006.01.02") + "/_mapping") if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error()) return } - err = json.Unmarshal(response, &m) + + err = json.Unmarshal(response, &fullm) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error()) return } - var j []byte - for _, v := range m { - j, _ = json.Marshal(v.Mappings.Properties) - break + for _, v := range fullm { + m = v.(map[string]interface{}) } + if mapping, hasMap := m["mappings"]; hasMap { + rt.flattenMap("", mapping.(map[string]interface{})["properties"].(map[string]interface{}), flatMap) + } + + j, _ := json.Marshal(flatMap) + w.Write(j) } @@ -562,20 +566,30 @@ func (rt *Router) ApiHandler(w http.ResponseWriter, r *http.Request) { */ ds, _ := time.Parse("2006-01-02 15:04:05 (MST)", request.Values.DateStart+" (MSK)") de, _ := time.Parse("2006-01-02 15:04:05 (MST)", request.Values.DateEnd+" (MSK)") + var use_source string + var query string + + if len(request.Values.Fields) == 0 { + use_source = `"_source": true,` + } else { + use_source = `"_source": false,` + } + if len(request.Values.Timefields) > 0 { + timefield := request.Values.Timefields[0] - query := `{ + query = `{ "sort": [ - {"timestamp": "desc"} + {"` + timefield + `": "desc"} ], - "_source": false, - "fields": [ "` + strings.Join(request.Values.Fields, "\", \"") + `" ], + ` + use_source + ` + "fields": ["` + timefield + `", "` + strings.Join(request.Values.Fields, "\", \"") + `" ], "query": { "bool": { "must": [], "filter": [ { "range": { - "timestamp": { + "` + timefield + `": { "gte": "` + ds.Format("2006-01-02T15:04:05.000Z") + `", "lte": "` + de.Format("2006-01-02T15:04:05.000Z") + `", "format": "strict_date_optional_time" @@ -588,6 +602,22 @@ func (rt *Router) ApiHandler(w http.ResponseWriter, r *http.Request) { } } }` + + } else { + query = `{ + "size": 500, + ` + use_source + ` + "fields": ["` + strings.Join(request.Values.Fields, "\", \"") + `" ], + "query": { + "bool": { + "must": [], + "filter": [], + "should": [], + "must_not": [] + } + } + }` + } log.Println(query) var req map[string]interface{} _ = json.Unmarshal([]byte(query), &req) @@ -597,7 +627,6 @@ func (rt *Router) ApiHandler(w http.ResponseWriter, r *http.Request) { log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error()) return } - //j, _ := json.Marshal(response) w.Write(response) }