Skip to content

Commit

Permalink
работа с index/mappings
Browse files Browse the repository at this point in the history
  • Loading branch information
Boris committed Oct 14, 2024
1 parent f03c632 commit 1daf604
Show file tree
Hide file tree
Showing 4 changed files with 123 additions and 39 deletions.
51 changes: 43 additions & 8 deletions front/search.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@
</div>
</form>
<div class="d-flex align-items-center invisible" id="loading"><strong>Loading...</strong><div class="spinner-border ml-auto" role="status" aria-hidden="true"></div></div>
<ul class="list-unstyled mb-0 overflow-auto" style="max-height: 1024px;" id="result"> </ul>
<span class="mb-0 overflow-auto" style="max-height: 1024px;" id="result"> </span>
</main><!-- /.main -->

</div><!-- /.row -->
Expand Down Expand Up @@ -103,7 +103,6 @@ <h5 class="modal-title" id="exampleModalLabel">X-tract indices from snapshot</h5
</div><!-- /.modal -->



</body>
<link rel="stylesheet" type="text/css" href="/assets/css/jquery.datetimepicker.min.css">
<script src="/assets/js/jquery-3.5.1.min.js"></script>
Expand All @@ -112,6 +111,7 @@ <h5 class="modal-title" id="exampleModalLabel">X-tract indices from snapshot</h5

<script>
const date = new Date();
var mapping = {}
date.setMinutes(date.getMinutes() - 15)
$.datetimepicker.setLocale('ru');
$('#datetimepicker_start').datetimepicker({timepicker: true, format:'Y-m-d H:i:s', step: 15, value:date});
Expand Down Expand Up @@ -207,7 +207,7 @@ <h5 class="modal-title" id="exampleModalLabel">X-tract indices from snapshot</h5
success: function (data) {
var str = "";
for (var k in data) {
str += "<li class='list-group-item'>"+ k + "(" + data[k].Type + ")" +"&nbsp;<input type='checkbox' name='fields' data-type='" + data[k].Type + "' value='" + k + "'></li>";
str += "<li class='list-group-item'><input type='checkbox' name='fields' data-type='" + data[k] + "' value='" + k + "'>&nbsp;"+ k + "(" + data[k] + ")" +"</li>";
}
$("#fields").html(str);
$("#loading").addClass('invisible');
Expand All @@ -218,11 +218,13 @@ <h5 class="modal-title" id="exampleModalLabel">X-tract indices from snapshot</h5

$("#search").click(function(){
$("#loading").removeClass('invisible');
$("#result").html("");
fields = []
tf = []
$("input[name='fields']").each(function() {
console.log(this)

if (this.dataset.type=="date") {
tf.push(this.value)
}
if (this.checked) {
fields.push(this.value)
}
Expand Down Expand Up @@ -250,9 +252,42 @@ <h5 class="modal-title" id="exampleModalLabel">X-tract indices from snapshot</h5
success: function (data) {
res = data.hits.hits
var str = "";
for (var k in res) {

str += "<li>"+ res[k].fields +"</li>";

if ( res.length > 0 ) {
str = "<table class='table'><thead><tr><th>Time</th>";
if ( fields.length == 0 ) {
str+="<th>_source</th>";
} else {
for (var f in res[0].fields) {
if (f==tf[0]) {
continue;
} else {
str+="<th>" + f + "</th>";
}

}
}
str+="</tr></thead><tbody>";
for (var k in res) {
str += "<tr>";
if ( fields.length == 0 ) {
str += "<td>"+ res[k]._source[tf[0]] +"</td>";
str += "<td>"+ JSON.stringify(res[k]._source); +"</td>";
} else {
str += "<td>"+ res[k].fields[tf[0]] +"</td>";
for (var f in res[k].fields) {
if (f==tf[0]) {
continue;
} else {
str+="<td>" + res[k].fields[f]+"</td>";
}
}
}
str+="</tr>";
}
str+="</tbody></table>";
} else {
str = "<h3>No search results found</h3>";
}
$("#result").html(str);
},
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.21
require (
github.com/jteeuwen/go-bindata v3.0.7+incompatible // indirect
github.com/uzhinskiy/lib.go v0.1.7
github.com/Jeffail/gabs/v2 v2.7.0
gopkg.in/yaml.v2 v2.3.0
github.com/opensearch-project/opensearch-go v1.1.0
)
19 changes: 19 additions & 0 deletions modules/router/methods.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,3 +233,22 @@ func (rt *Router) Barrel(ind_array IndicesInSnap, s3 bool) ([]string, []string)
}
return a, b
}

func (rt *Router) flattenMap(prefix string, nestedMap map[string]interface{}, flatMap map[string]string) {
for key, value := range nestedMap {
fullKey := key
if prefix != "" {
fullKey = prefix + "." + key
}

if subMap, ok := value.(map[string]interface{}); ok {
if typeVal, exists := subMap["type"]; exists {
flatMap[fullKey] = typeVal.(string)
}

if props, hasProps := subMap["properties"]; hasProps {
rt.flattenMap(fullKey, props.(map[string]interface{}), flatMap)
}
}
}
}
91 changes: 60 additions & 31 deletions modules/router/router.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (

"time"

//"github.com/Jeffail/gabs/v2"
"github.com/flant/elasticsearch-extractor/modules/config"
"github.com/flant/elasticsearch-extractor/modules/front"
"github.com/flant/elasticsearch-extractor/modules/version"
Expand All @@ -43,17 +44,18 @@ type Router struct {
type apiRequest struct {
Action string `json:"action,omitempty"` // Имя вызываемого метода*
Values struct {
Indices []string `json:"indices,omitempty"`
Repo string `json:"repo,omitempty"`
OrderDir string `json:"odir,omitempty"`
OrderType string `json:"otype,omitempty"`
Snapshot string `json:"snapshot,omitempty"`
Index string `json:"index,omitempty"`
Cluster string `json:"cluster,omitempty"`
Xql string `json:"xql,omitempty"`
Fields []string `json:"fields,omitempty"`
DateStart string `json:"date_start,omitempty"`
DateEnd string `json:"date_end,omitempty"`
Indices []string `json:"indices,omitempty"`
Repo string `json:"repo,omitempty"`
OrderDir string `json:"odir,omitempty"`
OrderType string `json:"otype,omitempty"`
Snapshot string `json:"snapshot,omitempty"`
Index string `json:"index,omitempty"`
Cluster string `json:"cluster,omitempty"`
Xql string `json:"xql,omitempty"`
Fields []string `json:"fields,omitempty"`
Timefields []string `json:"timefields,omitempty"`
DateStart string `json:"date_start,omitempty"`
DateEnd string `json:"date_end,omitempty"`
} `json:"values,omitempty"`
}

Expand Down Expand Up @@ -113,14 +115,6 @@ type indexGroup struct {
Index string `json:"index,omitempty"`
}

type IndexMapping map[string]struct {
Mappings struct {
Properties map[string]struct {
Type string
}
}
}

type Cluster struct {
Name string
Host string
Expand Down Expand Up @@ -518,25 +512,35 @@ func (rt *Router) ApiHandler(w http.ResponseWriter, r *http.Request) {
case "get_mapping":
{
t := time.Now()
var m IndexMapping
var (
fullm map[string]interface{}
m map[string]interface{}
)

flatMap := make(map[string]string)
response, err := rt.doGet(request.Values.Cluster + request.Values.Index + t.Format("2006.01.02") + "/_mapping")
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
return
}
err = json.Unmarshal(response, &m)

err = json.Unmarshal(response, &fullm)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
return
}
var j []byte
for _, v := range m {
j, _ = json.Marshal(v.Mappings.Properties)
break
for _, v := range fullm {
m = v.(map[string]interface{})
}

if mapping, hasMap := m["mappings"]; hasMap {
rt.flattenMap("", mapping.(map[string]interface{})["properties"].(map[string]interface{}), flatMap)
}

j, _ := json.Marshal(flatMap)

w.Write(j)
}

Expand All @@ -562,20 +566,30 @@ func (rt *Router) ApiHandler(w http.ResponseWriter, r *http.Request) {
*/
ds, _ := time.Parse("2006-01-02 15:04:05 (MST)", request.Values.DateStart+" (MSK)")
de, _ := time.Parse("2006-01-02 15:04:05 (MST)", request.Values.DateEnd+" (MSK)")
var use_source string
var query string

if len(request.Values.Fields) == 0 {
use_source = `"_source": true,`
} else {
use_source = `"_source": false,`
}
if len(request.Values.Timefields) > 0 {
timefield := request.Values.Timefields[0]

query := `{
query = `{
"sort": [
{"timestamp": "desc"}
{"` + timefield + `": "desc"}
],
"_source": false,
"fields": [ "` + strings.Join(request.Values.Fields, "\", \"") + `" ],
` + use_source + `
"fields": ["` + timefield + `", "` + strings.Join(request.Values.Fields, "\", \"") + `" ],
"query": {
"bool": {
"must": [],
"filter": [
{
"range": {
"timestamp": {
"` + timefield + `": {
"gte": "` + ds.Format("2006-01-02T15:04:05.000Z") + `",
"lte": "` + de.Format("2006-01-02T15:04:05.000Z") + `",
"format": "strict_date_optional_time"
Expand All @@ -588,6 +602,22 @@ func (rt *Router) ApiHandler(w http.ResponseWriter, r *http.Request) {
}
}
}`

} else {
query = `{
"size": 500,
` + use_source + `
"fields": ["` + strings.Join(request.Values.Fields, "\", \"") + `" ],
"query": {
"bool": {
"must": [],
"filter": [],
"should": [],
"must_not": []
}
}
}`
}
log.Println(query)
var req map[string]interface{}
_ = json.Unmarshal([]byte(query), &req)
Expand All @@ -597,7 +627,6 @@ func (rt *Router) ApiHandler(w http.ResponseWriter, r *http.Request) {
log.Println(remoteIP, "\t", r.Method, "\t", r.URL.Path, "\t", request.Action, "\t", http.StatusInternalServerError, "\t", err.Error())
return
}
//j, _ := json.Marshal(response)
w.Write(response)
}

Expand Down

0 comments on commit 1daf604

Please sign in to comment.