From 68983cff39cff019b8895b13c8b2606b9e00ea0e Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Wed, 6 Dec 2023 17:30:33 -0800 Subject: [PATCH 01/12] Updating Ip fields to use doc_values to search Signed-off-by: Harsha Vamsi Kalluri --- CHANGELOG.md | 1 + .../test/search/370_ip_field_doc_values.yml | 134 ++++++++++++++++++ .../index/mapper/IpFieldMapper.java | 69 +++++++-- 3 files changed, 192 insertions(+), 12 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/370_ip_field_doc_values.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index f2c0229e8c7fc..87dec87ac740d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -215,6 +215,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Ensure Jackson default maximums introduced in 2.16.0 do not conflict with OpenSearch settings ([#11890](https://github.com/opensearch-project/OpenSearch/pull/11890)) - Extract cluster management for integration tests into JUnit test rule out of OpenSearchIntegTestCase ([#11877](https://github.com/opensearch-project/OpenSearch/pull/11877)), ([#12000](https://github.com/opensearch-project/OpenSearch/pull/12000)) - Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2 ([#11968](https://github.com/opensearch-project/OpenSearch/pull/11968)) +- Updates IpField to be searchable when only `doc_values` are enabled ([#11508](https://github.com/opensearch-project/OpenSearch/pull/11508)) ### Deprecated diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_ip_field_doc_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_ip_field_doc_values.yml new file mode 100644 index 0000000000000..33419c5bb223c --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_ip_field_doc_values.yml @@ -0,0 +1,134 @@ +setup: + - skip: + features: [ "headers" ] + version: " - 2.11.99" + reason: "searching with only doc_values was added in 2.12.0" + +--- +"search on ip fields with doc_values enabled": + - do: + indices.create: + index: test-iodvq + body: + mappings: + dynamic: false + properties: + ip_index: + type: ip + doc_values: false + ip_doc_values: + type: ip + index: false + ip_both: + type: ip + index: true + doc_values: true + + + - do: + headers: + Content-Type: application/json + index: + index: "test-iodvq" + id: 1 + body: + ip_index: "192.168.0.1" + ip_doc_values: "192.168.0.2" + ip_both: "192.168.0.3" + + - do: + headers: + Content-Type: application/json + index: + index: "test-iodvq" + id: 2 + body: + ip_index: "192.168.1.1" + ip_doc_values: "192.168.1.2" + ip_both: "192.168.1.3" + + + - do: + headers: + Content-Type: application/json + index: + index: "test-iodvq" + id: 3 + body: + ip_index: "192.168.2.1" + ip_doc_values: "192.168.2.2" + ip_both: "192.168.2.3" + + + - do: + indices.refresh: {} + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + term: + ip_index: "192.168.0.1" + + - match: {hits.total: 1} + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + term: + ip_doc_values: "192.168.0.2" + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + term: + ip_both: "192.168.0.3" + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + terms: + ip_index: ["192.168.0.1", "192.168.1.1"] + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + range: + ip_doc_values: + gte: "192.168.0.2" + + - match: { hits.total: 3 } + + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + range: + ip_both: + gte: "192.168.0.2" + lte: "192.168.2.2" + + - match: { hits.total: 2 } diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index 2a677d8bc1352..4f1f703585fb0 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -36,7 +36,9 @@ import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; @@ -222,9 +224,10 @@ protected Object parseSourceValue(Object value) { @Override public Query termQuery(Object value, @Nullable QueryShardContext context) { - failIfNotIndexed(); + failIfNotIndexedAndNoDocValues(); + Query query; if (value instanceof InetAddress) { - return InetAddressPoint.newExactQuery(name(), (InetAddress) value); + query = InetAddressPoint.newExactQuery(name(), (InetAddress) value); } else { if (value instanceof BytesRef) { value = ((BytesRef) value).utf8ToString(); @@ -232,15 +235,39 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { String term = value.toString(); if (term.contains("/")) { final Tuple cidr = InetAddresses.parseCidr(term); - return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); + query = InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); } InetAddress address = InetAddresses.forString(term); - return InetAddressPoint.newExactQuery(name(), address); + query = InetAddressPoint.newExactQuery(name(), address); } + if (isSearchable() && hasDocValues()) { + return new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ); + } + if (hasDocValues()) { + // InetAddressPoint uses a rangeQuery internally for terms + return SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ); + } + return query; } @Override public Query termsQuery(List values, QueryShardContext context) { + failIfNotIndexedAndNoDocValues(); InetAddress[] addresses = new InetAddress[values.size()]; int i = 0; for (Object value : values) { @@ -265,14 +292,32 @@ public Query termsQuery(List values, QueryShardContext context) { @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { - failIfNotIndexed(); - return rangeQuery( - lowerTerm, - upperTerm, - includeLower, - includeUpper, - (lower, upper) -> InetAddressPoint.newRangeQuery(name(), lower, upper) - ); + failIfNotIndexedAndNoDocValues(); + return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (lower, upper) -> { + Query query = InetAddressPoint.newRangeQuery(name(), lower, upper); + if (isSearchable() && hasDocValues()) { + return new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ); + } + if (hasDocValues()) { + return SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ); + } + return query; + }); } /** From c34fed9b011aa9e4421e13c456e6fdd933c323f7 Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Thu, 7 Dec 2023 19:39:13 -0800 Subject: [PATCH 02/12] Fix IP tests Signed-off-by: Harsha Vamsi Kalluri --- .../index/mapper/IpFieldMapper.java | 5 +- .../index/mapper/IpFieldTypeTests.java | 208 ++++++++++++++++-- 2 files changed, 191 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index 4f1f703585fb0..5941cca8e6395 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -236,9 +236,10 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { if (term.contains("/")) { final Tuple cidr = InetAddresses.parseCidr(term); query = InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); + } else { + InetAddress address = InetAddresses.forString(term); + query = InetAddressPoint.newExactQuery(name(), address); } - InetAddress address = InetAddresses.forString(term); - query = InetAddressPoint.newExactQuery(name(), address); } if (isSearchable() && hasDocValues()) { return new IndexOrDocValuesQuery( diff --git a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java index 1a66037d98d71..56bd20f9232ee 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java @@ -32,10 +32,14 @@ package org.opensearch.index.mapper; import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.PointRangeQuery; +import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; @@ -75,22 +79,77 @@ public void testTermQuery() { MappedFieldType ft = new IpFieldMapper.IpFieldType("field"); String ip = "2001:db8::2:1"; - assertEquals(InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)), ft.termQuery(ip, null)); + + Query query = InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)); + + assertEquals( + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), + ft.termQuery(ip, null) + ); ip = "192.168.1.7"; - assertEquals(InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)), ft.termQuery(ip, null)); + query = InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)); + assertEquals( + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), + ft.termQuery(ip, null) + ); ip = "2001:db8::2:1"; String prefix = ip + "/64"; - assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64), ft.termQuery(prefix, null)); + + query = InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64); + assertEquals( + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), + ft.termQuery(prefix, null) + ); ip = "192.168.1.7"; prefix = ip + "/16"; - assertEquals(InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16), ft.termQuery(prefix, null)); + query = InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16); + assertEquals( + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), + ft.termQuery(prefix, null) + ); - MappedFieldType unsearchable = new IpFieldMapper.IpFieldType("field", false, false, true, null, Collections.emptyMap()); + MappedFieldType unsearchable = new IpFieldMapper.IpFieldType("field", false, false, false, null, Collections.emptyMap()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("::1", null)); - assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); } public void testTermsQuery() { @@ -118,44 +177,123 @@ public void testTermsQuery() { public void testRangeQuery() { MappedFieldType ft = new IpFieldMapper.IpFieldType("field"); - + Query query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddressPoint.MAX_VALUE); assertEquals( - InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddressPoint.MAX_VALUE), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), ft.rangeQuery(null, null, randomBoolean(), randomBoolean(), null, null, null, null) ); + query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddresses.forString("192.168.2.0")); assertEquals( - InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddresses.forString("192.168.2.0")), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), ft.rangeQuery(null, "192.168.2.0", randomBoolean(), true, null, null, null, null) ); + query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddresses.forString("192.168.1.255")); assertEquals( - InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddresses.forString("192.168.1.255")), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), ft.rangeQuery(null, "192.168.2.0", randomBoolean(), false, null, null, null, null) ); + query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::"), InetAddressPoint.MAX_VALUE); assertEquals( - InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::"), InetAddressPoint.MAX_VALUE), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), ft.rangeQuery("2001:db8::", null, true, randomBoolean(), null, null, null, null) ); + query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::1"), InetAddressPoint.MAX_VALUE); assertEquals( - InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::1"), InetAddressPoint.MAX_VALUE), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), ft.rangeQuery("2001:db8::", null, false, randomBoolean(), null, null, null, null) ); + query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::"), InetAddresses.forString("2001:db8::ffff")); assertEquals( - InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::"), InetAddresses.forString("2001:db8::ffff")), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), ft.rangeQuery("2001:db8::", "2001:db8::ffff", true, true, null, null, null, null) ); + query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::1"), InetAddresses.forString("2001:db8::fffe")); assertEquals( - InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::1"), InetAddresses.forString("2001:db8::fffe")), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), ft.rangeQuery("2001:db8::", "2001:db8::ffff", false, false, null, null, null, null) ); + query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::2"), InetAddresses.forString("2001:db8::")); assertEquals( - InetAddressPoint.newRangeQuery("field", InetAddresses.forString("2001:db8::2"), InetAddresses.forString("2001:db8::")), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), // same lo/hi values but inclusive=false so this won't match anything ft.rangeQuery("2001:db8::1", "2001:db8::1", false, false, null, null, null, null) ); @@ -178,30 +316,60 @@ public void testRangeQuery() { ) ); + query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddresses.forString("::fffe:ffff:ffff")); assertEquals( - InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::"), InetAddresses.forString("::fffe:ffff:ffff")), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), // same lo/hi values but inclusive=false so this won't match anything ft.rangeQuery("::", "0.0.0.0", true, false, null, null, null, null) ); + query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::1:0:0:0"), InetAddressPoint.MAX_VALUE); assertEquals( - InetAddressPoint.newRangeQuery("field", InetAddresses.forString("::1:0:0:0"), InetAddressPoint.MAX_VALUE), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), // same lo/hi values but inclusive=false so this won't match anything ft.rangeQuery("255.255.255.255", "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", false, true, null, null, null, null) ); + query = InetAddressPoint.newRangeQuery("field", InetAddresses.forString("192.168.1.7"), InetAddresses.forString("2001:db8::")); assertEquals( // lower bound is ipv4, upper bound is ipv6 - InetAddressPoint.newRangeQuery("field", InetAddresses.forString("192.168.1.7"), InetAddresses.forString("2001:db8::")), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowRangeQuery( + ((PointRangeQuery) query).getField(), + new BytesRef(((PointRangeQuery) query).getLowerPoint()), + new BytesRef(((PointRangeQuery) query).getUpperPoint()), + true, + true + ) + ), ft.rangeQuery("::ffff:c0a8:107", "2001:db8::", true, true, null, null, null, null) ); - MappedFieldType unsearchable = new IpFieldMapper.IpFieldType("field", false, false, true, null, Collections.emptyMap()); + MappedFieldType unsearchable = new IpFieldMapper.IpFieldType("field", false, false, false, null, Collections.emptyMap()); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, () -> unsearchable.rangeQuery("::1", "2001::", true, true, null, null, null, null) ); - assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); } public void testFetchSourceValue() throws IOException { From 6364147858c152b4a76ef89045b5336ae921eb53 Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Fri, 8 Dec 2023 10:12:06 -0800 Subject: [PATCH 03/12] Fix skip to allow yaml test to pass on main Signed-off-by: Harsha Vamsi Kalluri --- .../rest-api-spec/test/search/370_ip_field_doc_values.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_ip_field_doc_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_ip_field_doc_values.yml index 33419c5bb223c..21f095cbca4fc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_ip_field_doc_values.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_ip_field_doc_values.yml @@ -1,8 +1,8 @@ setup: - skip: features: [ "headers" ] - version: " - 2.11.99" - reason: "searching with only doc_values was added in 2.12.0" + version: " - 2.99.99" + reason: "searching with only doc_values was added in 3.0.0" --- "search on ip fields with doc_values enabled": From 4a3aff88c76bee26030facc1cb5bf4dbfc610bfc Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Mon, 8 Jan 2024 14:42:44 -0800 Subject: [PATCH 04/12] Update tests to use existing test file Signed-off-by: Harsha Vamsi Kalluri --- .../test/search/340_doc_values_field.yml | 127 ++++++++++++++--- .../test/search/370_ip_field_doc_values.yml | 134 ------------------ 2 files changed, 111 insertions(+), 150 deletions(-) delete mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/370_ip_field_doc_values.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml index c7b00d5fbbef2..e17388ae2d428 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml @@ -47,6 +47,10 @@ setup: type: unsigned_long index: true doc_values: true + ip_field: + type: ip + index: true + doc_values: true - do: bulk: @@ -54,11 +58,11 @@ setup: refresh: true body: - '{"index": {"_index": "test-iodvq", "_id": "1" }}' - - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800 }' + - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }' - '{ "index": { "_index": "test-iodvq", "_id": "2" }}' - - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801 }' + - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }' - '{ "index": { "_index": "test-iodvq", "_id": "3" } }' - - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802 }' + - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }' - do: search: @@ -162,7 +166,6 @@ setup: - match: { hits.total: 1 } - - do: search: rest_total_hits_as_int: true @@ -174,6 +177,16 @@ setup: - match: { hits.total: 1 } + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + term: + ip_field: "192.168.0.1" + + - match: {hits.total: 1} - do: search: @@ -186,7 +199,6 @@ setup: - match: { hits.total: 2 } - - do: search: rest_total_hits_as_int: true @@ -264,6 +276,17 @@ setup: - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + terms: + ip_field: ["192.168.0.1", "192.168.0.2"] + + - match: { hits.total: 2 } + - do: search: rest_total_hits_as_int: true @@ -384,6 +407,19 @@ setup: - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + range: + ip_field: + gte: "192.168.0.1" + lte: "192.168.0.2" + + - match: { hits.total: 2 } + --- "search on fields with only index enabled": - do: @@ -428,6 +464,10 @@ setup: type: unsigned_long index: true doc_values: false + ip_field: + type: ip + index: true + doc_values: false - do: bulk: @@ -435,11 +475,11 @@ setup: refresh: true body: - '{"index": {"_index": "test-index", "_id": "1" }}' - - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800 }' + - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }' - '{ "index": { "_index": "test-index", "_id": "2" }}' - - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801 }' + - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }' - '{ "index": { "_index": "test-index", "_id": "3" } }' - - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802 }' + - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }' - do: search: @@ -465,7 +505,6 @@ setup: - match: { hits.total: 2 } - - do: search: rest_total_hits_as_int: true @@ -555,6 +594,16 @@ setup: - match: { hits.total: 1 } + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + term: + ip_field: "192.168.0.1" + + - match: {hits.total: 1} - do: search: @@ -567,7 +616,6 @@ setup: - match: { hits.total: 2 } - - do: search: rest_total_hits_as_int: true @@ -645,6 +693,17 @@ setup: - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + terms: + ip_field: ["192.168.0.1", "192.168.0.2"] + + - match: { hits.total: 2 } + - do: search: rest_total_hits_as_int: true @@ -765,6 +824,18 @@ setup: - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + range: + ip_field: + gte: "192.168.0.1" + lte: "192.168.0.2" + + - match: { hits.total: 2 } --- "search on fields with only doc_values enabled": - do: @@ -809,6 +880,10 @@ setup: type: unsigned_long index: false doc_values: true + ip_field: + type: ip + index: false + doc_values: true - do: bulk: @@ -816,11 +891,11 @@ setup: refresh: true body: - '{"index": {"_index": "test-doc-values", "_id": "1" }}' - - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800 }' + - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }' - '{ "index": { "_index": "test-doc-values", "_id": "2" }}' - - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801 }' + - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }' - '{ "index": { "_index": "test-doc-values", "_id": "3" } }' - - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802 }' + - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }' - do: search: @@ -846,7 +921,6 @@ setup: - match: { hits.total: 2 } - - do: search: rest_total_hits_as_int: true @@ -924,7 +998,6 @@ setup: - match: { hits.total: 1 } - - do: search: rest_total_hits_as_int: true @@ -936,6 +1009,16 @@ setup: - match: { hits.total: 1 } + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + term: + ip_field: "192.168.0.3" + + - match: { hits.total: 1 } - do: search: @@ -948,7 +1031,6 @@ setup: - match: { hits.total: 2 } - - do: search: rest_total_hits_as_int: true @@ -1145,3 +1227,16 @@ setup: } - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + range: + ip_field: + gte: "192.168.0.1" + lte: "192.168.0.2" + + - match: { hits.total: 2 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_ip_field_doc_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_ip_field_doc_values.yml deleted file mode 100644 index 21f095cbca4fc..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/370_ip_field_doc_values.yml +++ /dev/null @@ -1,134 +0,0 @@ -setup: - - skip: - features: [ "headers" ] - version: " - 2.99.99" - reason: "searching with only doc_values was added in 3.0.0" - ---- -"search on ip fields with doc_values enabled": - - do: - indices.create: - index: test-iodvq - body: - mappings: - dynamic: false - properties: - ip_index: - type: ip - doc_values: false - ip_doc_values: - type: ip - index: false - ip_both: - type: ip - index: true - doc_values: true - - - - do: - headers: - Content-Type: application/json - index: - index: "test-iodvq" - id: 1 - body: - ip_index: "192.168.0.1" - ip_doc_values: "192.168.0.2" - ip_both: "192.168.0.3" - - - do: - headers: - Content-Type: application/json - index: - index: "test-iodvq" - id: 2 - body: - ip_index: "192.168.1.1" - ip_doc_values: "192.168.1.2" - ip_both: "192.168.1.3" - - - - do: - headers: - Content-Type: application/json - index: - index: "test-iodvq" - id: 3 - body: - ip_index: "192.168.2.1" - ip_doc_values: "192.168.2.2" - ip_both: "192.168.2.3" - - - - do: - indices.refresh: {} - - - do: - search: - rest_total_hits_as_int: true - index: test-iodvq - body: - query: - term: - ip_index: "192.168.0.1" - - - match: {hits.total: 1} - - - do: - search: - rest_total_hits_as_int: true - index: test-iodvq - body: - query: - term: - ip_doc_values: "192.168.0.2" - - - match: { hits.total: 1 } - - - do: - search: - rest_total_hits_as_int: true - index: test-iodvq - body: - query: - term: - ip_both: "192.168.0.3" - - - match: { hits.total: 1 } - - - do: - search: - rest_total_hits_as_int: true - index: test-iodvq - body: - query: - terms: - ip_index: ["192.168.0.1", "192.168.1.1"] - - - match: { hits.total: 2 } - - - do: - search: - rest_total_hits_as_int: true - index: test-iodvq - body: - query: - range: - ip_doc_values: - gte: "192.168.0.2" - - - match: { hits.total: 3 } - - - - do: - search: - rest_total_hits_as_int: true - index: test-iodvq - body: - query: - range: - ip_both: - gte: "192.168.0.2" - lte: "192.168.2.2" - - - match: { hits.total: 2 } From 8e661c0bab059aa32458cf59094170930c7bc776 Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Tue, 9 Jan 2024 16:38:20 -0800 Subject: [PATCH 05/12] Changing skip version to match bwc Signed-off-by: Harsha Vamsi Kalluri --- .../rest-api-spec/test/search/340_doc_values_field.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml index e17388ae2d428..993b333bdb508 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml @@ -1,8 +1,8 @@ setup: - skip: features: [ "headers" ] - version: " - 2.11.99" - reason: "searching with only doc_values was added in 2.12.0" + version: " - 2.99.99" + reason: "searching with only doc_values was added in 3.0.0" --- "search on fields with both index and doc_values enabled": - do: From 3ae54b687dcde04ef0dfbab53fb6867b263efc81 Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Thu, 18 Jan 2024 21:24:53 -0800 Subject: [PATCH 06/12] Using exact match instead of range Signed-off-by: Harsha Vamsi Kalluri --- .../opensearch/index/mapper/IpFieldMapper.java | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index 5941cca8e6395..05bbad5a675c3 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -40,6 +40,7 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.opensearch.Version; @@ -244,24 +245,11 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { if (isSearchable() && hasDocValues()) { return new IndexOrDocValuesQuery( query, - SortedSetDocValuesField.newSlowRangeQuery( - ((PointRangeQuery) query).getField(), - new BytesRef(((PointRangeQuery) query).getLowerPoint()), - new BytesRef(((PointRangeQuery) query).getUpperPoint()), - true, - true - ) + SortedSetDocValuesField.newSlowExactQuery(name(), ((BytesRef) value)) ); } if (hasDocValues()) { - // InetAddressPoint uses a rangeQuery internally for terms - return SortedSetDocValuesField.newSlowRangeQuery( - ((PointRangeQuery) query).getField(), - new BytesRef(((PointRangeQuery) query).getLowerPoint()), - new BytesRef(((PointRangeQuery) query).getUpperPoint()), - true, - true - ); + return SortedSetDocValuesField.newSlowExactQuery(name(), ((BytesRef) value)); } return query; } From 89876e011d9b9781e98fbc3eb842a9911164c709 Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Fri, 19 Jan 2024 09:39:51 -0800 Subject: [PATCH 07/12] Spotless Signed-off-by: Harsha Vamsi Kalluri --- .../java/org/opensearch/index/mapper/IpFieldMapper.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index 05bbad5a675c3..3f7cdc4b55169 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -40,7 +40,6 @@ import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.opensearch.Version; @@ -243,10 +242,7 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { } } if (isSearchable() && hasDocValues()) { - return new IndexOrDocValuesQuery( - query, - SortedSetDocValuesField.newSlowExactQuery(name(), ((BytesRef) value)) - ); + return new IndexOrDocValuesQuery(query, SortedSetDocValuesField.newSlowExactQuery(name(), ((BytesRef) value))); } if (hasDocValues()) { return SortedSetDocValuesField.newSlowExactQuery(name(), ((BytesRef) value)); From 3d6f4ccf81974f8472e9f81dde4d8cdde432c721 Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Fri, 19 Jan 2024 10:58:31 -0800 Subject: [PATCH 08/12] Fix IP field tests Signed-off-by: Harsha Vamsi Kalluri --- .../index/mapper/IpFieldMapper.java | 9 ++++- .../index/mapper/IpFieldTypeTests.java | 38 ++----------------- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index 3f7cdc4b55169..c9b6f07e84e40 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -242,7 +242,14 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { } } if (isSearchable() && hasDocValues()) { - return new IndexOrDocValuesQuery(query, SortedSetDocValuesField.newSlowExactQuery(name(), ((BytesRef) value))); + String term = value.toString(); + if (value instanceof String) + value = new BytesRef(((String) value).getBytes()); + if (term.contains("/")) { + final Tuple cidr = InetAddresses.parseCidr(term); + return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); + } + return new IndexOrDocValuesQuery(query, SortedSetDocValuesField.newSlowExactQuery(name(), (BytesRef)(value))); } if (hasDocValues()) { return SortedSetDocValuesField.newSlowExactQuery(name(), ((BytesRef) value)); diff --git a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java index 56bd20f9232ee..ddf3fc45c11ef 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java @@ -85,13 +85,7 @@ public void testTermQuery() { assertEquals( new IndexOrDocValuesQuery( query, - SortedSetDocValuesField.newSlowRangeQuery( - ((PointRangeQuery) query).getField(), - new BytesRef(((PointRangeQuery) query).getLowerPoint()), - new BytesRef(((PointRangeQuery) query).getUpperPoint()), - true, - true - ) + SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(ip)) ), ft.termQuery(ip, null) ); @@ -101,13 +95,7 @@ public void testTermQuery() { assertEquals( new IndexOrDocValuesQuery( query, - SortedSetDocValuesField.newSlowRangeQuery( - ((PointRangeQuery) query).getField(), - new BytesRef(((PointRangeQuery) query).getLowerPoint()), - new BytesRef(((PointRangeQuery) query).getUpperPoint()), - true, - true - ) + SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(ip)) ), ft.termQuery(ip, null) ); @@ -117,16 +105,7 @@ public void testTermQuery() { query = InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64); assertEquals( - new IndexOrDocValuesQuery( - query, - SortedSetDocValuesField.newSlowRangeQuery( - ((PointRangeQuery) query).getField(), - new BytesRef(((PointRangeQuery) query).getLowerPoint()), - new BytesRef(((PointRangeQuery) query).getUpperPoint()), - true, - true - ) - ), + query, ft.termQuery(prefix, null) ); @@ -134,16 +113,7 @@ public void testTermQuery() { prefix = ip + "/16"; query = InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16); assertEquals( - new IndexOrDocValuesQuery( - query, - SortedSetDocValuesField.newSlowRangeQuery( - ((PointRangeQuery) query).getField(), - new BytesRef(((PointRangeQuery) query).getLowerPoint()), - new BytesRef(((PointRangeQuery) query).getUpperPoint()), - true, - true - ) - ), + query, ft.termQuery(prefix, null) ); From a84588e69020a2204fd2eb926ccc82c31ed5336e Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Fri, 19 Jan 2024 14:18:53 -0800 Subject: [PATCH 09/12] Fix spotless + precommit failure Signed-off-by: Harsha Vamsi Kalluri --- .../index/mapper/IpFieldMapper.java | 11 +++++++--- .../index/mapper/IpFieldTypeTests.java | 20 ++++--------------- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index c9b6f07e84e40..c1db6b0415519 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -243,15 +243,20 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { } if (isSearchable() && hasDocValues()) { String term = value.toString(); - if (value instanceof String) - value = new BytesRef(((String) value).getBytes()); + if (value instanceof String) value = new BytesRef(((String) value)); if (term.contains("/")) { final Tuple cidr = InetAddresses.parseCidr(term); return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); } - return new IndexOrDocValuesQuery(query, SortedSetDocValuesField.newSlowExactQuery(name(), (BytesRef)(value))); + return new IndexOrDocValuesQuery(query, SortedSetDocValuesField.newSlowExactQuery(name(), (BytesRef) (value))); } if (hasDocValues()) { + String term = value.toString(); + if (value instanceof String) value = new BytesRef(((String) value)); + if (term.contains("/")) { + final Tuple cidr = InetAddresses.parseCidr(term); + return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); + } return SortedSetDocValuesField.newSlowExactQuery(name(), ((BytesRef) value)); } return query; diff --git a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java index ddf3fc45c11ef..e848700850800 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java @@ -83,20 +83,14 @@ public void testTermQuery() { Query query = InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)); assertEquals( - new IndexOrDocValuesQuery( - query, - SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(ip)) - ), + new IndexOrDocValuesQuery(query, SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(ip))), ft.termQuery(ip, null) ); ip = "192.168.1.7"; query = InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)); assertEquals( - new IndexOrDocValuesQuery( - query, - SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(ip)) - ), + new IndexOrDocValuesQuery(query, SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(ip))), ft.termQuery(ip, null) ); @@ -104,18 +98,12 @@ public void testTermQuery() { String prefix = ip + "/64"; query = InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 64); - assertEquals( - query, - ft.termQuery(prefix, null) - ); + assertEquals(query, ft.termQuery(prefix, null)); ip = "192.168.1.7"; prefix = ip + "/16"; query = InetAddressPoint.newPrefixQuery("field", InetAddresses.forString(ip), 16); - assertEquals( - query, - ft.termQuery(prefix, null) - ); + assertEquals(query, ft.termQuery(prefix, null)); MappedFieldType unsearchable = new IpFieldMapper.IpFieldType("field", false, false, false, null, Collections.emptyMap()); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("::1", null)); From d455bee2ee42b11ff335fc08348e234a0d7d34eb Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Mon, 22 Jan 2024 11:57:46 -0800 Subject: [PATCH 10/12] Get point out of query and into value Signed-off-by: Harsha Vamsi Kalluri --- .../java/org/opensearch/index/mapper/IpFieldMapper.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java index c1db6b0415519..db8da8a949d6f 100644 --- a/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java @@ -243,21 +243,22 @@ public Query termQuery(Object value, @Nullable QueryShardContext context) { } if (isSearchable() && hasDocValues()) { String term = value.toString(); - if (value instanceof String) value = new BytesRef(((String) value)); if (term.contains("/")) { final Tuple cidr = InetAddresses.parseCidr(term); return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); } - return new IndexOrDocValuesQuery(query, SortedSetDocValuesField.newSlowExactQuery(name(), (BytesRef) (value))); + return new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint())) + ); } if (hasDocValues()) { String term = value.toString(); - if (value instanceof String) value = new BytesRef(((String) value)); if (term.contains("/")) { final Tuple cidr = InetAddresses.parseCidr(term); return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2()); } - return SortedSetDocValuesField.newSlowExactQuery(name(), ((BytesRef) value)); + return SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint())); } return query; } From 468313ad767acf32e1905bc6d0921fb12d23f8fb Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Mon, 22 Jan 2024 13:00:10 -0800 Subject: [PATCH 11/12] Fix term tests Signed-off-by: Harsha Vamsi Kalluri --- .../org/opensearch/index/mapper/IpFieldTypeTests.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java index e848700850800..0a2435553b19e 100644 --- a/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/IpFieldTypeTests.java @@ -83,14 +83,20 @@ public void testTermQuery() { Query query = InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)); assertEquals( - new IndexOrDocValuesQuery(query, SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(ip))), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(((PointRangeQuery) query).getLowerPoint())) + ), ft.termQuery(ip, null) ); ip = "192.168.1.7"; query = InetAddressPoint.newExactQuery("field", InetAddresses.forString(ip)); assertEquals( - new IndexOrDocValuesQuery(query, SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(ip))), + new IndexOrDocValuesQuery( + query, + SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef(((PointRangeQuery) query).getLowerPoint())) + ), ft.termQuery(ip, null) ); From 407b7239521a37963c474fc8c3ca5d1ac5fd3a4b Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Mon, 29 Jan 2024 09:54:41 -0800 Subject: [PATCH 12/12] Add skip test logic to only doc_values test Signed-off-by: Harsha Vamsi Kalluri --- .../rest-api-spec/test/search/340_doc_values_field.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml index 993b333bdb508..d5ece1719dc48 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml @@ -1,8 +1,3 @@ -setup: - - skip: - features: [ "headers" ] - version: " - 2.99.99" - reason: "searching with only doc_values was added in 3.0.0" --- "search on fields with both index and doc_values enabled": - do: @@ -838,6 +833,10 @@ setup: - match: { hits.total: 2 } --- "search on fields with only doc_values enabled": + - skip: + features: [ "headers" ] + version: " - 2.99.99" + reason: "searching with only doc_values was added in 3.0.0" - do: indices.create: index: test-doc-values