Skip to content

Commit

Permalink
feat(query) remove cutoff_frequency (#1657)
Browse files Browse the repository at this point in the history 
* Remove cutoff_frequency, deprecated in es7.3.0, forbidden in es8

From https://www.elastic.co/guide/en/elasticsearch/reference/8.8/migrating-8.0.html

    The cutoff_frequency parameter has been removed from the match and
    multi_match query.

    Details
    The cutoff_frequency parameter, deprecated in 7.x, has been removed
    in 8.0 from match and multi_match queries. The same functionality
    can be achieved without any configuration provided that the total
    number of hits is not tracked.

    Impact
    Discontinue use of the cutoff_frequency parameter. Search requests
    containing this parameter in a match or multi_match query will
    return an error.

Note in the above "...provided that the total number of hits is not
tracked".

`track_total_hits` does not appear in the pelias codebases, so we
shouldn't have any issues there.

* update to pelias-query w/o cutoff_frequency

* fixup! Remove cutoff_frequency, deprecated in es7.3.0, forbidden in es8
  • Loading branch information
@michaelkirk
michaelkirk authored Apr 19, 2024
1 parent 05aa871 commit 24d3306
Show file tree
Hide file tree
Showing 5 changed files with 1 addition and 44 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
"pelias-microservice-wrapper": "^1.10.0",
"pelias-model": "^9.0.0",
"pelias-parser": "2.2.0",
"pelias-query": "^11.0.0",
"pelias-query": "^11.2.0",
"pelias-sorting": "^1.7.0",
"predicates": "^2.0.0",
"regenerate": "^1.4.0",
Expand Down
23 changes: 0 additions & 23 deletions query/autocomplete_defaults.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,11 @@ module.exports = _.merge({}, peliasQuery.defaults, {
'ngram:analyzer': 'peliasQuery',
'ngram:field': 'name.default',
'ngram:boost': 100,
'ngram:cutoff_frequency': 0.01,

'phrase:analyzer': 'peliasQuery',
'phrase:field': 'phrase.default',
'phrase:boost': 1,
'phrase:slop': 3,
'phrase:cutoff_frequency': 0.01,

'focus:function': 'exp',
'focus:offset': '0km',
Expand All @@ -40,37 +38,28 @@ module.exports = _.merge({}, peliasQuery.defaults, {
'address:housenumber:analyzer': 'peliasHousenumber',
'address:housenumber:field': 'address_parts.number',
'address:housenumber:boost': 2,
'address:housenumber:cutoff_frequency': 0.01,

'address:street:analyzer': 'peliasQuery',
'address:street:field': 'address_parts.street',
'address:street:boost': 1,
'address:street:cutoff_frequency': 0.01,

'address:cross_street:analyzer': 'peliasQuery',
'address:cross_street:field': 'address_parts.cross_street',
'address:cross_street:boost': 5,
'address:cross_street:cutoff_frequency': 0.01,

'address:postcode:analyzer': 'peliasZip',
'address:postcode:field': 'address_parts.zip',
'address:postcode:boost': 2000,
'address:postcode:cutoff_frequency': 0.01,

// generic multi_match config
'multi_match:type': 'cross_fields',
'multi_match:ngrams_strict:type': 'phrase',
'multi_match:first_tokens_only:type': 'phrase',
'multi_match:boost_exact_matches:type': 'phrase',

// setting 'cutoff_frequency' will result in very common
// terms such as country not scoring at all
// 'multi_match:cutoff_frequency': 0.01,

'admin:country_a:analyzer': 'standard',
'admin:country_a:field': 'parent.country_a.ngram',
'admin:country_a:boost': 1,
'admin:country_a:cutoff_frequency': 0.01,

// these options affect the `boundary.country` hard filter
'multi_match:boundary_country:analyzer': 'standard',
Expand All @@ -79,62 +68,50 @@ module.exports = _.merge({}, peliasQuery.defaults, {
'admin:country:analyzer': 'peliasAdmin',
'admin:country:field': 'parent.country.ngram',
'admin:country:boost': 1,
'admin:country:cutoff_frequency': 0.01,

'admin:dependency:analyzer': 'peliasAdmin',
'admin:dependency:field': 'parent.dependency.ngram',
'admin:dependency:boost': 1,
'admin:dependency:cutoff_frequency': 0.01,

'admin:region:analyzer': 'peliasAdmin',
'admin:region:field': 'parent.region.ngram',
'admin:region:boost': 1,
'admin:region:cutoff_frequency': 0.01,

'admin:region_a:analyzer': 'peliasAdmin',
'admin:region_a:field': 'parent.region_a.ngram',
'admin:region_a:boost': 1,
'admin:region_a:cutoff_frequency': 0.01,

'admin:macroregion:analyzer': 'peliasAdmin',
'admin:macroregion:field': 'parent.macroregion.ngram',
'admin:macroregion:boost': 1,
'admin:macroregion:cutoff_frequency': 0.01,

'admin:county:analyzer': 'peliasAdmin',
'admin:county:field': 'parent.county.ngram',
'admin:county:boost': 1,
'admin:county:cutoff_frequency': 0.01,

'admin:macrocounty:analyzer': 'peliasAdmin',
'admin:macrocounty:field': 'parent.macrocounty.ngram',
'admin:macrocounty:boost': 1,
'admin:macrocounty:cutoff_frequency': 0.01,

'admin:localadmin:analyzer': 'peliasAdmin',
'admin:localadmin:field': 'parent.localadmin.ngram',
'admin:localadmin:boost': 1,
'admin:localadmin:cutoff_frequency': 0.01,

'admin:locality:analyzer': 'peliasAdmin',
'admin:locality:field': 'parent.locality.ngram',
'admin:locality:boost': 1,
'admin:locality:cutoff_frequency': 0.01,

'admin:locality_a:analyzer': 'peliasAdmin',
'admin:locality_a:field': 'parent.locality_a.ngram',
'admin:locality_a:boost': 1,
'admin:locality_a:cutoff_frequency': 0.01,

'admin:neighbourhood:analyzer': 'peliasAdmin',
'admin:neighbourhood:field': 'parent.neighbourhood.ngram',
'admin:neighbourhood:boost': 1,
'admin:neighbourhood:cutoff_frequency': 0.01,

'admin:borough:analyzer': 'peliasAdmin',
'admin:borough:field': 'parent.borough.ngram',
'admin:borough:boost': 1,
'admin:borough:cutoff_frequency': 0.01,

// an additional 'name' field to add to admin multi-match queries.
// this is used to improve venue matching in cases where the we
Expand Down
15 changes: 0 additions & 15 deletions query/search_defaults.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ module.exports = _.merge({}, peliasQuery.defaults, {
'ngram:analyzer': 'peliasQuery',
'ngram:field': 'name.default',
'ngram:boost': 1,
'ngram:cutoff_frequency': 0.01,
'ngram:minimum_should_match': '1<-1 3<-25%',

'match:main:analyzer': 'peliasQuery',
Expand All @@ -43,28 +42,22 @@ module.exports = _.merge({}, peliasQuery.defaults, {
'address:housenumber:analyzer': 'peliasHousenumber',
'address:housenumber:field': 'address_parts.number',
'address:housenumber:boost': 2,
'address:housenumber:cutoff_frequency': 0.01,

'address:street:analyzer': 'peliasQuery',
'address:street:field': 'address_parts.street',
'address:street:boost': 5,
'address:street:slop': 4,
'address:street:cutoff_frequency': 0.01,

'address:postcode:analyzer': 'peliasZip',
'address:postcode:field': 'address_parts.zip',
'address:postcode:boost': 20,
'address:postcode:cutoff_frequency': 0.01,

// multi match query views require 'type' to be specified
'multi_match:type': 'best_fields',
// generic multi_match cutoff_frequency
'multi_match:cutoff_frequency': 0.01,

'admin:country_a:analyzer': 'standard',
'admin:country_a:field': 'parent.country_a',
'admin:country_a:boost': 1,
'admin:country_a:cutoff_frequency': 0.01,

// these config variables are used for the 'boundary.country' hard filter
'multi_match:boundary_country:analyzer': 'standard',
Expand All @@ -73,42 +66,34 @@ module.exports = _.merge({}, peliasQuery.defaults, {
'admin:country:analyzer': 'peliasAdmin',
'admin:country:field': 'parent.country',
'admin:country:boost': 1,
'admin:country:cutoff_frequency': 0.01,

'admin:region:analyzer': 'peliasAdmin',
'admin:region:field': 'parent.region',
'admin:region:boost': 1,
'admin:region:cutoff_frequency': 0.01,

'admin:region_a:analyzer': 'peliasAdmin',
'admin:region_a:field': 'parent.region_a',
'admin:region_a:boost': 1,
'admin:region_a:cutoff_frequency': 0.01,

'admin:county:analyzer': 'peliasAdmin',
'admin:county:field': 'parent.county',
'admin:county:boost': 1,
'admin:county:cutoff_frequency': 0.01,

'admin:localadmin:analyzer': 'peliasAdmin',
'admin:localadmin:field': 'parent.localadmin',
'admin:localadmin:boost': 1,
'admin:localadmin:cutoff_frequency': 0.01,

'admin:locality:analyzer': 'peliasAdmin',
'admin:locality:field': 'parent.locality',
'admin:locality:boost': 1,
'admin:locality:cutoff_frequency': 0.01,

'admin:borough:analyzer': 'peliasAdmin',
'admin:borough:field': 'parent.borough',
'admin:borough:boost': 1,
'admin:borough:cutoff_frequency': 0.01,

'admin:neighbourhood:analyzer': 'peliasAdmin',
'admin:neighbourhood:field': 'parent.neighbourhood',
'admin:neighbourhood:boost': 1,
'admin:neighbourhood:cutoff_frequency': 0.01,

'popularity:field': 'popularity',
'popularity:modifier': 'log1p',
Expand Down
3 changes: 0 additions & 3 deletions test/unit/fixture/search_pelias_parser_full_address.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@ module.exports = {
'match': {
'address_parts.number': {
'query': '123',
'cutoff_frequency': 0.01,
'boost': vs['address:housenumber:boost'],
'analyzer': vs['address:housenumber:analyzer']
}
Expand All @@ -69,7 +68,6 @@ module.exports = {
'match': {
'address_parts.street': {
'query': 'main st',
'cutoff_frequency': 0.01,
'boost': vs['address:street:boost'],
'analyzer': vs['address:street:analyzer']
}
Expand All @@ -78,7 +76,6 @@ module.exports = {
'match': {
'address_parts.zip': {
'query': '10010',
'cutoff_frequency': 0.01,
'boost': vs['address:postcode:boost'],
'analyzer': vs['address:postcode:analyzer']
}
Expand Down
2 changes: 0 additions & 2 deletions test/unit/fixture/search_pelias_parser_regions_address.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ module.exports = {
'match': {
'address_parts.number': {
'query': '1',
'cutoff_frequency': 0.01,
'boost': vs['address:housenumber:boost'],
'analyzer': vs['address:housenumber:analyzer']
}
Expand All @@ -68,7 +67,6 @@ module.exports = {
'match': {
'address_parts.street': {
'query': 'water st',
'cutoff_frequency': 0.01,
'boost': vs['address:street:boost'],
'analyzer': vs['address:street:analyzer']
}
Expand Down

0 comments on commit 24d3306

Please sign in to comment.