diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d6a953ba..1b17c8f63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.10...2.x) ### Features ### Enhancements -- Add graph creation stats to the KNNStats API. [#1141](https://github.com/opensearch-project/k-NN/pull/1141) +* Added support for ignore_unmapped in KNN queries. [#1071](https://github.com/opensearch-project/k-NN/pull/1071) +* Add graph creation stats to the KNNStats API. [#1141](https://github.com/opensearch-project/k-NN/pull/1141) ### Bug Fixes ### Infrastructure ### Documentation diff --git a/src/main/java/org/opensearch/knn/index/IndexUtil.java b/src/main/java/org/opensearch/knn/index/IndexUtil.java index 60156b4a7..574e4a977 100644 --- a/src/main/java/org/opensearch/knn/index/IndexUtil.java +++ b/src/main/java/org/opensearch/knn/index/IndexUtil.java @@ -13,6 +13,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import org.opensearch.Version; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.MappingMetadata; import org.opensearch.common.ValidationException; @@ -24,6 +25,7 @@ import java.io.File; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import static org.opensearch.knn.common.KNNConstants.BYTES_PER_KILOBYTES; @@ -32,6 +34,13 @@ public class IndexUtil { + private static final Version MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED = Version.V_2_11_0; + private static final Map minimalRequiredVersionMap = new HashMap() { + { + put("ignore_unmapped", MINIMAL_SUPPORTED_VERSION_FOR_IGNORE_UNMAPPED); + } + }; + /** * Determines the size of a file on disk in kilobytes * @@ -195,4 +204,12 @@ public static Map getParametersAtLoading(SpaceType spaceType, KN return Collections.unmodifiableMap(loadParameters); } + + public static boolean isClusterOnOrAfterMinRequiredVersion(String key) { + Version minimalRequiredVersion = minimalRequiredVersionMap.get(key); + if (minimalRequiredVersion == null) { + return false; + } + return KNNClusterUtil.instance().getClusterMinVersion().onOrAfter(minimalRequiredVersion); + } } diff --git a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java index 88c5c30f1..10c652627 100644 --- a/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java +++ b/src/main/java/org/opensearch/knn/index/query/KNNQueryBuilder.java @@ -6,6 +6,7 @@ package org.opensearch.knn.index.query; import lombok.extern.log4j.Log4j2; +import org.apache.lucene.search.MatchNoDocsQuery; import org.opensearch.core.common.Strings; import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.query.QueryBuilder; @@ -31,6 +32,7 @@ import java.util.List; import java.util.Objects; +import static org.opensearch.knn.index.IndexUtil.isClusterOnOrAfterMinRequiredVersion; import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.validateByteVectorValue; /** @@ -43,6 +45,7 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { public static final ParseField VECTOR_FIELD = new ParseField("vector"); public static final ParseField K_FIELD = new ParseField("k"); public static final ParseField FILTER_FIELD = new ParseField("filter"); + public static final ParseField IGNORE_UNMAPPED_FIELD = new ParseField("ignore_unmapped"); public static int K_MAX = 10000; /** * The name for the knn query @@ -55,6 +58,7 @@ public class KNNQueryBuilder extends AbstractQueryBuilder { private final float[] vector; private int k = 0; private QueryBuilder filter; + private boolean ignoreUnmapped = false; /** * Constructs a new knn query @@ -88,6 +92,7 @@ public KNNQueryBuilder(String fieldName, float[] vector, int k, QueryBuilder fil this.vector = vector; this.k = k; this.filter = filter; + this.ignoreUnmapped = false; } public static void initialize(ModelDao modelDao) { @@ -113,6 +118,9 @@ public KNNQueryBuilder(StreamInput in) throws IOException { vector = in.readFloatArray(); k = in.readInt(); filter = in.readOptionalNamedWriteable(QueryBuilder.class); + if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { + ignoreUnmapped = in.readOptionalBoolean(); + } } catch (IOException ex) { throw new RuntimeException("[KNN] Unable to create KNNQueryBuilder", ex); } @@ -126,6 +134,7 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep QueryBuilder filter = null; String queryName = null; String currentFieldName = null; + boolean ignoreUnmapped = false; XContentParser.Token token; KNNCounter.KNN_QUERY_REQUESTS.increment(); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -134,6 +143,8 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep } else if (token == XContentParser.Token.START_OBJECT) { throwParsingExceptionOnMultipleFields(NAME, parser.getTokenLocation(), fieldName, currentFieldName); fieldName = currentFieldName; + System.out.println(currentFieldName); + System.out.println(IGNORE_UNMAPPED_FIELD.getPreferredName()); while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); @@ -144,6 +155,10 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep boost = parser.floatValue(); } else if (K_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { k = (Integer) NumberFieldMapper.NumberType.INTEGER.parse(parser.objectBytes(), false); + } else if (IGNORE_UNMAPPED_FIELD.getPreferredName().equals(currentFieldName)) { + if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { + ignoreUnmapped = parser.booleanValue(); + } } else if (AbstractQueryBuilder.NAME_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { queryName = parser.text(); } else { @@ -176,6 +191,7 @@ public static KNNQueryBuilder fromXContent(XContentParser parser) throws IOExcep } KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(fieldName, ObjectsToFloats(vector), k, filter); + knnQueryBuilder.ignoreUnmapped(ignoreUnmapped); knnQueryBuilder.queryName(queryName); knnQueryBuilder.boost(boost); return knnQueryBuilder; @@ -187,6 +203,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeFloatArray(vector); out.writeInt(k); out.writeOptionalNamedWriteable(filter); + if (isClusterOnOrAfterMinRequiredVersion("ignore_unmapped")) { + out.writeOptionalBoolean(ignoreUnmapped); + } } /** @@ -211,6 +230,20 @@ public QueryBuilder getFilter() { return this.filter; } + /** + * Sets whether the query builder should ignore unmapped paths (and run a + * {@link MatchNoDocsQuery} in place of this query) or throw an exception if + * the path is unmapped. + */ + public KNNQueryBuilder ignoreUnmapped(boolean ignoreUnmapped) { + this.ignoreUnmapped = ignoreUnmapped; + return this; + } + + public boolean getIgnoreUnmapped() { + return this.ignoreUnmapped; + } + @Override public void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); @@ -221,6 +254,9 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio if (filter != null) { builder.field(FILTER_FIELD.getPreferredName(), filter); } + if (ignoreUnmapped) { + builder.field(IGNORE_UNMAPPED_FIELD.getPreferredName(), ignoreUnmapped); + } printBoostAndQueryName(builder); builder.endObject(); builder.endObject(); @@ -230,6 +266,10 @@ public void doXContent(XContentBuilder builder, Params params) throws IOExceptio protected Query doToQuery(QueryShardContext context) { MappedFieldType mappedFieldType = context.fieldMapper(this.fieldName); + if (mappedFieldType == null && ignoreUnmapped) { + return new MatchNoDocsQuery(); + } + if (!(mappedFieldType instanceof KNNVectorFieldMapper.KNNVectorFieldType)) { throw new IllegalArgumentException(String.format("Field '%s' is not knn_vector type.", this.fieldName)); } diff --git a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java index 4e7f739a7..e540725fc 100644 --- a/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java +++ b/src/test/java/org/opensearch/knn/index/query/KNNQueryBuilderTests.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.opensearch.Version; import org.opensearch.cluster.ClusterModule; @@ -41,6 +42,7 @@ import java.util.List; import java.util.Optional; +import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -307,4 +309,16 @@ private void assertSerialization(final Version version, final Optional knnQueryBuilder.doToQuery(mock(QueryShardContext.class))); + } }