Skip to content

Commit

Permalink
Merge pull request #1257 from nyaruka/es_raw
Browse files Browse the repository at this point in the history
Replace deprecated ES library with set of shortcut functions
  • Loading branch information
rowanseymour authored May 17, 2024
2 parents 7823ac3 + 0c8234f commit ae97c3f
Show file tree
Hide file tree
Showing 8 changed files with 219 additions and 158 deletions.
221 changes: 90 additions & 131 deletions contactql/es/query.go

Large diffs are not rendered by default.

5 changes: 1 addition & 4 deletions contactql/es/query_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,7 @@ func TestElasticQuery(t *testing.T) {
query := es.ToElasticQuery(env, mapper, parsed)
assert.NotNil(t, query, tc.Description)

source, err := query.Source()
require.NoError(t, err, "error requesting source for elastic query in ", testName)

asJSON, err := jsonx.Marshal(source)
asJSON, err := jsonx.Marshal(query)
require.NoError(t, err)

test.AssertEqualJSON(t, tc.Elastic, asJSON, "elastic mismatch in %s", testName)
Expand Down
16 changes: 6 additions & 10 deletions contactql/es/sort.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,15 @@ import (

"github.com/nyaruka/goflow/assets"
"github.com/nyaruka/goflow/contactql"

"github.com/olivere/elastic/v7"
"github.com/nyaruka/goflow/utils/elastic"
"github.com/pkg/errors"
)

// ToElasticFieldSort returns the elastic FieldSort for the passed in sort by string
func ToElasticFieldSort(sortBy string, resolver contactql.Resolver) (*elastic.FieldSort, error) {
func ToElasticFieldSort(sortBy string, resolver contactql.Resolver) (elastic.Sort, error) {
// default to most recent first by id
if sortBy == "" {
return elastic.NewFieldSort("id").Desc(), nil
return elastic.SortBy("id", false), nil
}

// figure out if we are ascending or descending (default is ascending, can be changed with leading -)
Expand All @@ -30,12 +29,12 @@ func ToElasticFieldSort(sortBy string, resolver contactql.Resolver) (*elastic.Fi

// name needs to be sorted by keyword field
if property == contactql.AttributeName {
return elastic.NewFieldSort("name.keyword").Order(ascending), nil
return elastic.SortBy("name.keyword", ascending), nil
}

// other attributes are straight sorts
if property == contactql.AttributeID || property == contactql.AttributeCreatedOn || property == contactql.AttributeLastSeenOn || property == contactql.AttributeLanguage {
return elastic.NewFieldSort(property).Order(ascending), nil
return elastic.SortBy(property, ascending), nil
}

// we are sorting by a custom field
Expand All @@ -52,8 +51,5 @@ func ToElasticFieldSort(sortBy string, resolver contactql.Resolver) (*elastic.Fi
key = fmt.Sprintf("fields.%s", field.Type())
}

sort := elastic.NewFieldSort(key)
sort = sort.Nested(elastic.NewNestedSort("fields").Filter(elastic.NewTermQuery("fields.field", field.UUID())))
sort = sort.Order(ascending)
return sort, nil
return elastic.SortNested(key, elastic.Term("fields.field", field.UUID()), "fields", ascending), nil
}
3 changes: 1 addition & 2 deletions contactql/es/sort_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,7 @@ func TestElasticSort(t *testing.T) {
if tc.Error != "" {
assert.EqualError(t, err, tc.Error)
} else {
src, _ := sort.Source()
encoded := jsonx.MustMarshal(src)
encoded := jsonx.MustMarshal(sort)
test.AssertEqualJSON(t, []byte(tc.Elastic), encoded, "field sort mismatch for %s", tc.Description)
}
}
Expand Down
3 changes: 0 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ require (
github.com/buger/jsonparser v1.1.1
github.com/go-playground/validator/v10 v10.20.0
github.com/nyaruka/gocommon v1.54.9
github.com/olivere/elastic/v7 v7.0.32
github.com/pkg/errors v0.9.1
github.com/sergi/go-diff v1.3.1
github.com/shopspring/decimal v1.4.0
Expand All @@ -28,9 +27,7 @@ require (
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/gofrs/uuid v4.4.0+incompatible // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/nyaruka/null/v2 v2.0.3 // indirect
github.com/nyaruka/phonenumbers v1.3.5 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
Expand Down
8 changes: 0 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx2
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s=
Expand All @@ -33,8 +31,6 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
Expand All @@ -44,16 +40,12 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/nyaruka/gocommon v1.54.9 h1:BvWgXQc7h9Qlhj347N56wPZiik64SGmv+qSLmar4pVo=
github.com/nyaruka/gocommon v1.54.9/go.mod h1:rWkEIpYIK98zL9Qm6PeMXJ+84WcWlArf01RfuWWCYvQ=
github.com/nyaruka/null/v2 v2.0.3 h1:rdmMRQyVzrOF3Jff/gpU/7BDR9mQX0lcLl4yImsA3kw=
github.com/nyaruka/null/v2 v2.0.3/go.mod h1:OCVeCkCXwrg5/qE6RU0c1oUVZBy+ZDrT+xYg1XSaIWA=
github.com/nyaruka/phonenumbers v1.3.5 h1:WZLbQn61j2E1OFnvpUTYbK/6hViUgl6tppJ55/E2iQM=
github.com/nyaruka/phonenumbers v1.3.5/go.mod h1:Ut+eFwikULbmCenH6InMKL9csUNLyxHuBLyfkpum11s=
github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand Down
98 changes: 98 additions & 0 deletions utils/elastic/query.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package elastic

type Query map[string]any

// Any is a shortcut for a bool query with a should clause
func Any(queries ...Query) Query {
return Query{"bool": Query{"should": queries}}
}

// All is a shortcut for a bool query with a must clause
func All(queries ...Query) Query {
return Query{"bool": Query{"must": queries}}
}

// Not is a shortcut for a bool query with a must_not clause
func Not(query Query) Query {
return Query{"bool": Query{"must_not": query}}
}

// Not is a shortcut for an ids query
func Ids(values ...string) Query {
return Query{"ids": Query{"values": values}}
}

// Term is a shortcut for a term query
func Term(field string, value any) Query {
return Query{"term": Query{field: value}}
}

// Exists is a shortcut for an exists query
func Exists(field string) Query {
return Query{"exists": Query{"field": field}}
}

// Nested is a shortcut for a nested query
func Nested(path string, query Query) Query {
return Query{"nested": Query{"path": path, "query": query}}
}

// Match is a shortcut for a match query
func Match(field string, value any) Query {
return Query{"match": Query{field: Query{"query": value}}}
}

// MatchPhrase is a shortcut for a match_phrase query
func MatchPhrase(field, value string) Query {
return Query{"match_phrase": Query{field: Query{"query": value}}}
}

// GreaterThan is a shortcut for a range query where x > value
func GreaterThan(field string, value any) Query {
return Query{"range": Query{field: Query{
"from": value,
"include_lower": false,
"include_upper": true,
"to": nil,
}}}
}

// GreaterThanOrEqual is a shortcut for a range query where x >= value
func GreaterThanOrEqual(field string, value any) Query {
return Query{"range": Query{field: Query{
"from": value,
"include_lower": true,
"include_upper": true,
"to": nil,
}}}
}

// LessThan is a shortcut for a range query where x < value
func LessThan(field string, value any) Query {
return Query{"range": Query{field: Query{
"from": nil,
"include_lower": true,
"include_upper": false,
"to": value,
}}}
}

// LessThanOrEqual is a shortcut for a range query where x <= value
func LessThanOrEqual(field string, value any) Query {
return Query{"range": Query{field: Query{
"from": nil,
"include_lower": true,
"include_upper": true,
"to": value,
}}}
}

// Between is a shortcut for a range query where from <= x < to
func Between(field string, from, to any) Query {
return Query{"range": Query{field: Query{
"from": from,
"include_lower": true,
"include_upper": false,
"to": to,
}}}
}
23 changes: 23 additions & 0 deletions utils/elastic/sort.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package elastic

type Sort map[string]any

// SortBy is a shortcut for a simple field sort
func SortBy(field string, ascending bool) Sort {
return Sort{field: map[string]any{"order": order(ascending)}}
}

// SortNested is a shortcut for a nested field sort
func SortNested(field string, filter Query, path string, ascending bool) Sort {
return Sort{field: map[string]any{
"nested": map[string]any{"filter": filter, "path": path},
"order": order(ascending),
}}
}

func order(asc bool) string {
if asc {
return "asc"
}
return "desc"
}

0 comments on commit ae97c3f

Please sign in to comment.