Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix compiling error by adding meta fields checker before 7DOT8 in legacy SQL #2701

Closed
wants to merge 1 commit into from
Closed

Fix compiling error by adding meta fields checker before 7DOT8 in legacy SQL #2701

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions legacy/src/main/java/org/opensearch/sql/legacy/executor/join/ElasticJoinExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import org.opensearch.common.unit.TimeValue;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.rest.BytesRestResponse;
import org.opensearch.rest.RestChannel;
import org.opensearch.search.SearchHit;
Expand All @@ -39,6 +38,7 @@
import org.opensearch.sql.legacy.query.join.NestedLoopsElasticRequestBuilder;
import org.opensearch.sql.legacy.query.join.TableInJoinRequestBuilder;
import org.opensearch.sql.legacy.query.planner.HashJoinQueryPlanRequestBuilder;
import org.opensearch.sql.legacy.utils.Util;

/** Created by Eliran on 15/9/2015. */
public abstract class ElasticJoinExecutor implements ElasticHitsExecutor {
Expand Down Expand Up @@ -223,9 +223,7 @@ protected SearchHit createUnmachedResult(
hit.getFields()
.forEach(
(fieldName, docField) ->
(MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName)
? metaFields
: documentFields)
(Util.META_FIELDS_BEFORE_7DOT8.contains(fieldName) ? metaFields : documentFields)
.put(fieldName, docField));
SearchHit searchHit = new SearchHit(docId, unmatchedId, documentFields, metaFields);

Expand Down
6 changes: 3 additions & 3 deletions legacy/src/main/java/org/opensearch/sql/legacy/executor/join/HashJoinElasticExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.opensearch.client.Client;
import org.opensearch.common.document.DocumentField;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.index.query.BoolQueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.search.SearchHit;
Expand All @@ -32,6 +31,7 @@
import org.opensearch.sql.legacy.query.join.HashJoinElasticRequestBuilder;
import org.opensearch.sql.legacy.query.join.TableInJoinRequestBuilder;
import org.opensearch.sql.legacy.query.maker.QueryMaker;
import org.opensearch.sql.legacy.utils.Util;

/** Created by Eliran on 22/8/2015. */
public class HashJoinElasticExecutor extends ElasticJoinExecutor {
Expand Down Expand Up @@ -187,7 +187,7 @@ private List<SearchHit> createCombinedResults(TableInJoinRequestBuilder secondTa
.getFields()
.forEach(
(fieldName, docField) ->
(MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName)
(Util.META_FIELDS_BEFORE_7DOT8.contains(fieldName)
? metaFields
: documentFields)
.put(fieldName, docField));
Expand Down Expand Up @@ -261,7 +261,7 @@ private void createKeyToResultsAndFillOptimizationStructure(
hit.getFields()
.forEach(
(fieldName, docField) ->
(MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName)
(Util.META_FIELDS_BEFORE_7DOT8.contains(fieldName)
? metaFields
: documentFields)
.put(fieldName, docField));
Expand Down
6 changes: 2 additions & 4 deletions legacy/src/main/java/org/opensearch/sql/legacy/executor/join/NestedLoopsElasticExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import org.opensearch.client.Client;
import org.opensearch.common.document.DocumentField;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.search.SearchHit;
import org.opensearch.search.SearchHits;
import org.opensearch.sql.legacy.domain.Condition;
Expand All @@ -32,6 +31,7 @@
import org.opensearch.sql.legacy.query.join.NestedLoopsElasticRequestBuilder;
import org.opensearch.sql.legacy.query.join.TableInJoinRequestBuilder;
import org.opensearch.sql.legacy.query.maker.Maker;
import org.opensearch.sql.legacy.utils.Util;

/** Created by Eliran on 15/9/2015. */
public class NestedLoopsElasticExecutor extends ElasticJoinExecutor {
Expand Down Expand Up @@ -200,9 +200,7 @@ private SearchHit getMergedHit(
.getFields()
.forEach(
(fieldName, docField) ->
(MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName)
? metaFields
: documentFields)
(Util.META_FIELDS_BEFORE_7DOT8.contains(fieldName) ? metaFields : documentFields)
.put(fieldName, docField));
SearchHit searchHit =
new SearchHit(
Expand Down
9 changes: 2 additions & 7 deletions legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/MinusExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import org.opensearch.client.Client;
import org.opensearch.common.document.DocumentField;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.search.SearchHit;
import org.opensearch.search.SearchHits;
import org.opensearch.sql.legacy.domain.Condition;
Expand Down Expand Up @@ -125,9 +124,7 @@ private void fillMinusHitsFromOneField(
.getFields()
.forEach(
(field, docField) ->
(MapperService.META_FIELDS_BEFORE_7DOT8.contains(field)
? metaFields
: documentFields)
(Util.META_FIELDS_BEFORE_7DOT8.contains(field) ? metaFields : documentFields)
.put(field, docField));
SearchHit searchHit = new SearchHit(currentId, currentId + "", documentFields, metaFields);
searchHit.sourceRef(someHit.getSourceRef());
Expand Down Expand Up @@ -157,9 +154,7 @@ private void fillMinusHitsFromResults(Set<ComperableHitResult> comperableHitResu
.getFields()
.forEach(
(fieldName, docField) ->
(MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName)
? metaFields
: documentFields)
(Util.META_FIELDS_BEFORE_7DOT8.contains(fieldName) ? metaFields : documentFields)
.put(fieldName, docField));
SearchHit searchHit =
new SearchHit(currentId, originalHit.getId(), documentFields, metaFields);
Expand Down
5 changes: 1 addition & 4 deletions legacy/src/main/java/org/opensearch/sql/legacy/executor/multi/UnionExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
import org.opensearch.action.search.SearchResponse;
import org.opensearch.client.Client;
import org.opensearch.common.document.DocumentField;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.search.SearchHit;
import org.opensearch.search.SearchHits;
import org.opensearch.sql.legacy.executor.ElasticHitsExecutor;
Expand Down Expand Up @@ -60,9 +59,7 @@ private void fillInternalSearchHits(
hit.getFields()
.forEach(
(fieldName, docField) ->
(MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName)
? metaFields
: documentFields)
(Util.META_FIELDS_BEFORE_7DOT8.contains(fieldName) ? metaFields : documentFields)
.put(fieldName, docField));
SearchHit searchHit = new SearchHit(currentId, hit.getId(), documentFields, metaFields);
searchHit.sourceRef(hit.getSourceRef());
Expand Down
6 changes: 2 additions & 4 deletions .../main/java/org/opensearch/sql/legacy/query/planner/physical/node/scroll/SearchHitRow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
import java.util.HashMap;
import java.util.Map;
import org.opensearch.common.document.DocumentField;
import org.opensearch.index.mapper.MapperService;
import org.opensearch.search.SearchHit;
import org.opensearch.sql.legacy.query.planner.physical.Row;
import org.opensearch.sql.legacy.utils.Util;

/**
*
Expand Down Expand Up @@ -146,9 +146,7 @@ private SearchHit cloneHit(Row<SearchHit> other) {
hit.getFields()
.forEach(
(fieldName, docField) ->
(MapperService.META_FIELDS_BEFORE_7DOT8.contains(fieldName)
? metaFields
: documentFields)
(Util.META_FIELDS_BEFORE_7DOT8.contains(fieldName) ? metaFields : documentFields)
.put(fieldName, docField));
SearchHit combined =
new SearchHit(
Expand Down
15 changes: 15 additions & 0 deletions legacy/src/main/java/org/opensearch/sql/legacy/utils/Util.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
import com.alibaba.druid.sql.parser.ParserException;
import com.alibaba.druid.sql.parser.SQLExprParser;
import com.alibaba.druid.sql.parser.Token;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
Expand All @@ -41,6 +43,19 @@ public class Util {

public static final String NESTED_JOIN_TYPE = "NestedJoinType";

public static final Set<String> META_FIELDS_BEFORE_7DOT8 =
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. Can we wrap this with a static method so we can change implementation easily?
  2. If we go ahead with this approach, please add comment regarding this is not ideal solution.
  3. What does the suffix 7D0T8 mean?

Copy link
Member

@cwperks cwperks May 31, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

7DOT8 is referring to ES 7.8.

"_ttl" and "_timestamp" were removed in ES6. See https://www.elastic.co/blog/elasticsearch-5-0-0-alpha4-released

"_type" was renamed to "_nested_path" in OS 2: opensearch-project/OpenSearch#3196

"_size" is not a built-in metadata mapper, it comes from the mapper-size plugin which is not installed by default. Using mapperService.isMetadataField("_size") should return true if the mapper-size plugin is installed in the cluster.

The other values in this list still exist and are built-in metadata fields so mapperService.isMetadataField() should return true.

Collections.unmodifiableSet(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: It's simpler to use ImmutableSet.of

new HashSet<>(
Arrays.asList(
"_id",
"_ignored",
"_index",
"_routing",
"_size",
"_timestamp",
"_ttl",
"_type")));

public static String joiner(List<KVValue> lists, String oper) {

if (lists.size() == 0) {
Expand Down
Loading