diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index f03acbbbfd..9b1ee21c7d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -57,6 +57,7 @@ import static org.opensearch.sql.legacy.TestUtils.getPhraseIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getResponseBody; import static org.opensearch.sql.legacy.TestUtils.getStringIndexMapping; +import static org.opensearch.sql.legacy.TestUtils.getDataTextKeywordIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getWeblogsIndexMapping; import static org.opensearch.sql.legacy.TestUtils.isIndexExist; import static org.opensearch.sql.legacy.TestUtils.loadDataByRestClient; @@ -584,7 +585,11 @@ public enum Index { CALCS(TestsConstants.TEST_INDEX_CALCS, "calcs", getMappingFile("calcs_index_mappings.json"), - "src/test/resources/calcs.json"),; + "src/test/resources/calcs.json"), + TEXTKEYWORD(TestsConstants.TEST_INDEX_TEXTKEYWORD, + "textkeyword", + getMappingFile("text_keyword_index_mapping.json"), + "src/test/resources/text_keyword_index.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 8f8ee4a70f..5ed447c4be 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -243,6 +243,11 @@ public static String getDataTypeNonnumericIndexMapping() { return getMappingFile(mappingFile); } + public static String getDataTextKeywordIndexMapping() { + String mappingFile = "text_keyword_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static void loadBulk(Client client, String jsonPath, String defaultIndex) throws Exception { System.out.println(String.format("Loading file %s into opensearch cluster", jsonPath)); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index a9f81c68fe..bfc6f5a681 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -53,6 +53,7 @@ public class TestsConstants { public final static String TEST_INDEX_BEER = TEST_INDEX + "_beer"; public final static String TEST_INDEX_NULL_MISSING = TEST_INDEX + "_null_missing"; public final static String TEST_INDEX_CALCS = TEST_INDEX + "_calcs"; + public final static String TEST_INDEX_TEXTKEYWORD = TEST_INDEX + "_textkeyword"; public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/TextTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/TextTypeIT.java new file mode 100644 index 0000000000..881eb02420 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/sql/TextTypeIT.java @@ -0,0 +1,258 @@ + /* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.sql; + + import org.junit.Test; + import org.opensearch.sql.legacy.SQLIntegTestCase; + + import java.io.IOException; + + import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TEXTKEYWORD; + import static org.opensearch.sql.util.MatcherUtils.schema; + import static org.opensearch.sql.util.MatcherUtils.verifySchema; + + public class TextTypeIT extends SQLIntegTestCase { + + + @Override + public void init() throws Exception { + super.init(); + loadIndex(Index.TEXTKEYWORD); + loadIndex(Index.CALCS); + + } + + // Select + + @Test + public void textKeywordTest() { + var result = executeJdbcRequest(String.format("select typeKeyword from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("typeKeyword", null, "keyword")); + } + + @Test + public void aggregateOnText() { + var result = executeJdbcRequest(String.format("select sum(int0) from %s GROUP BY typeText", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("sum(int0)", null, "integer")); + } + + @Test + public void aggregateOnKeyword() { + var result = executeJdbcRequest(String.format("select sum(int0) from %s GROUP BY typeKeyword", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("sum(int0)", null, "integer")); + } + + @Test + public void aggregateOnTextFieldData() { + var result = executeJdbcRequest(String.format("select sum(int0) from %s GROUP BY typeTextFieldData", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("sum(int0)", null, "integer")); + } + + @Test + public void aggregateOnKeywordFieldData() { + var result = executeJdbcRequest(String.format("select sum(int0) from %s GROUP BY typeKeywordFieldNoFieldData", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("sum(int0)", null, "integer")); + } + + @Test + public void aggregateOnTextAndFieldDataNoFields() { + var result = executeJdbcRequest(String.format("select sum(int0) from %s GROUP BY textDataFieldNoFields", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("sum(int0)", null, "integer")); + } + + // Where like + + @Test + public void whereLikeKeyword() { + var result = executeJdbcRequest(String.format("select typeKeyword from %s WHERE typeKeyword LIKE \\\"key*\\\"", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("typeKeyword", null, "keyword")); + } + + @Test + public void whereLikeText() { + var result = executeJdbcRequest(String.format("select typeText from %s WHERE typeText LIKE \\\"text*\\\"", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("typeText", null, "text")); + } + + @Test + public void whereLikeKeywordFieldNoFieldData() { + var result = executeJdbcRequest(String.format("select typeKeywordFieldNoFieldData from %s WHERE typeKeywordFieldNoFieldData LIKE \\\"keyword*\\\"", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("typeKeywordFieldNoFieldData", null, "text")); + } + + @Test + public void whereLikeTextFieldData() { + var result = executeJdbcRequest(String.format("select typeTextFieldData from %s WHERE typeTextFieldData LIKE \\\"keyFD*\\\"", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("typeTextFieldData", null, "text")); + } + + @Test + public void whereLiketextDataFieldNoFields() { + var result = executeJdbcRequest(String.format("select textDataFieldNoFields from %s WHERE textDataFieldNoFields LIKE \\\"textFDNF*\\\"", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("textDataFieldNoFields", null, "text")); + } + + // Wildcard + + @Test + public void whereWildcardKeyword() { + var result = executeJdbcRequest(String.format("select typeKeyword from %s WHERE wildcard_query(typeKeyword, \\\"key*\\\")", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("typeKeyword", null, "keyword")); + } + + @Test + public void whereWildcardText() { + var result = executeJdbcRequest(String.format("select typeText from %s WHERE wildcard_query(\\\"typeText\\\", \\\"text*\\\")", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("typeText", null, "text")); + } + + @Test + public void whereWildcardKeywordFieldNoFieldData() { + var result = executeJdbcRequest(String.format("select typeKeywordFieldNoFieldData from %s WHERE wildcard_query(\\\"typeKeywordFieldNoFieldData\\\", \\\"keyword*\\\")", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("typeKeywordFieldNoFieldData", null, "text")); + } + + @Test + public void whereWildcardTextFieldData() { + var result = executeJdbcRequest(String.format("select typeTextFieldData from %s WHERE wildcard_query(\\\"typeTextFieldData\\\", \\\"keyFD*\\\")", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("typeTextFieldData", null, "text")); + } + + @Test + public void whereWildcardtextDataFieldNoFields() { + var result = executeJdbcRequest(String.format("select textDataFieldNoFields from %s WHERE wildcard_query(\\\"textDataFieldNoFields\\\", \\\"textFDNF*\\\")", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("textDataFieldNoFields", null, "text")); + } + + // Locate + + @Test + public void selectLocateKeyword() { + var result = executeJdbcRequest(String.format("select locate(\\\"key*\\\", typeKeyword) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("locate(\"key*\", typeKeyword)", null, "integer")); + } + + @Test + public void selectLocateText() { + var result = executeJdbcRequest(String.format("select locate(\\\"text*\\\", typeText) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("locate(\"text*\", typeText)", null, "integer")); + } + + @Test + public void selectLocateTextKeywordFieldNoFieldData() { + var result = executeJdbcRequest(String.format("select locate(\\\"keyword*\\\", typeKeywordFieldNoFieldData) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("locate(\"keyword*\", typeKeywordFieldNoFieldData)", null, "integer")); + } + + @Test + public void selectLocateTypeTextFieldData() { + var result = executeJdbcRequest(String.format("select locate(\\\"keyFD*\\\", typeTextFieldData) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("locate(\"keyFD*\", typeTextFieldData)", null, "integer")); + } + + @Test + public void selectLocateTextDataFieldNoFields() { + var result = executeJdbcRequest(String.format("select locate(\\\"textFDNF*\\\", textDataFieldNoFields) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("locate(\"textFDNF*\", textDataFieldNoFields)", null, "integer")); + } + + // Position + + @Test + public void selectPositionKeyword() { + var result = executeJdbcRequest(String.format("select POSITION(\\\"key\\\" IN typeKeyword) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("LOCATE('key', typeKeyword)", null, "double")); + } + + @Test + public void selectPositionText() throws IOException { + var result = executeQuery(String.format("select POSITION(\\\"text\\\" IN typeText) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("typeText", null, "double")); + +// } + } + + @Test + public void selectPositionTextKeywordFieldNoFieldData() { + var result = executeJdbcRequest(String.format("select POSITION(\\\"keyword\\\" IN typeKeywordFieldNoFieldData) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("LOCATE('keyword', typeKeywordFieldNoFieldData)", null, "double")); + } + + @Test + public void selectPositionTypeTextFieldData() throws IOException { + var result = executeQuery(String.format("select POSITION(\\\"keyFD\\\" IN typeTextFieldData) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("LOCATE('keyFD', typeTextFieldData)", null, "double")); + } + + @Test + public void selectPositionTextDataFieldNoFields() { + var result = executeJdbcRequest(String.format("select POSITION(\\\"textFDNF\\\" IN textDataFieldNoFields) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("LOCATE('textFDNF', textDataFieldNoFields)", null, "double")); + } + + // Substring + + @Test + public void selectSubstringKeyword() { + var result = executeJdbcRequest(String.format("select SUBSTRING(typeKeyword, 1, 1) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("SUBSTRING(typeKeyword, 1, 1)", null, "keyword")); + } + + @Test + public void selectSubstringText() { + var result = executeJdbcRequest(String.format("select SUBSTRING(typeText, 1, 1) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("SUBSTRING(typeText, 1, 1)", null, "keyword")); + } + + @Test + public void selectSubstringTextKeywordFieldNoFieldData() { + var result = executeJdbcRequest(String.format("select SUBSTRING(typeKeywordFieldNoFieldData, 1, 1) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("SUBSTRING(typeKeywordFieldNoFieldData, 1, 1)", null, "keyword")); + } + + @Test + public void selectSubstringTypeTextFieldData() { + var result = executeJdbcRequest(String.format("select SUBSTRING(typeTextFieldData, 1, 1) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("SUBSTRING(typeTextFieldData, 1, 1)", null, "keyword")); + } + + @Test + public void selectSubstringTextDataFieldNoFields() { + var result = executeJdbcRequest(String.format("select SUBSTRING(textDataFieldNoFields, 1, 1) from %s", TEST_INDEX_TEXTKEYWORD)); + verifySchema(result, + schema("SUBSTRING(textDataFieldNoFields, 1, 1)", null, "keyword")); + } + } diff --git a/integ-test/src/test/resources/indexDefinitions/calcs_index_mappings.json b/integ-test/src/test/resources/indexDefinitions/calcs_index_mappings.json index 08a88a9d32..84bad2a0a0 100644 --- a/integ-test/src/test/resources/indexDefinitions/calcs_index_mappings.json +++ b/integ-test/src/test/resources/indexDefinitions/calcs_index_mappings.json @@ -20,7 +20,7 @@ "type" : "double" }, "str0" : { - "type" : "keyword" + "type" : "integer" }, "str1" : { "type" : "keyword" diff --git a/integ-test/src/test/resources/indexDefinitions/text_keyword_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/text_keyword_index_mapping.json new file mode 100644 index 0000000000..1975a10bcc --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/text_keyword_index_mapping.json @@ -0,0 +1,37 @@ +{ + "mappings" : { + "properties" : { + "typeKeyword" : { + "type" : "keyword" + }, + "typeText" : { + "type" : "text" + }, + "typeKeywordFieldNoFieldData" : { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 10 + } + } }, + "typeTextFieldData" : { + "type": "text", + "fielddata": true, + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 10 + } + } + }, + "textDataFieldNoFields" : { + "type": "text", + "fielddata": true + }, + "int0" : { + "type": "integer" + } + } + } +} diff --git a/integ-test/src/test/resources/text_keyword_index.json b/integ-test/src/test/resources/text_keyword_index.json new file mode 100644 index 0000000000..b9cb83d438 --- /dev/null +++ b/integ-test/src/test/resources/text_keyword_index.json @@ -0,0 +1,16 @@ +{"index": {"_id":"1"}} +{"typeKeyword": "key00", "typeText": "text00", "typeKeywordFieldNoFieldData": "keyword00","typeTextFieldData": "keyFD00", "typeKeywordFieldData": "textFD00", "textDataFieldNoFields": "textFDNF00","int0": 0} +{"index": {"_id":"2"}} +{"typeKeyword": "key01", "typeText": "text01", "typeKeywordFieldNoFieldData": "keyword01", "typeTextFieldData": "keyFD01", "typeKeywordFieldData": "textFD01OverTen", "textDataFieldNoFields": "textFDNF01", "int0": 1} +{"index": {"_id":"3"}} +{"typeKeyword": "key02", "typeText": "text02", "typeKeywordFieldNoFieldData": "keyword02", "typeTextFieldData": "keyFD02", "typeKeywordFieldData": "textFD02", "textDataFieldNoFields": "textFDNF02", "int0": 2} +{"index": {"_id":"4"}} +{"typeKeyword": "key03", "typeText": "text03", "typeKeywordFieldNoFieldData": "keyword03", "typeTextFieldData": "keyFD03OverTen", "typeKeywordFieldData": "textFD03", "textDataFieldNoFields": "textFDNF03", "int0": 3} +{"index": {"_id":"5"}} +{"typeKeyword": "key04", "typeText": "text04", "typeKeywordFieldNoFieldData": "keyword04", "typeTextFieldData": "keyFD04", "typeKeywordFieldData": "textFD04", "textDataFieldNoFields": "textFDNF04", "int0": 4} +{"index": {"_id":"6"}} +{"typeKeyword": "key05", "typeText": "text05", "typeKeywordFieldNoFieldData": "keyword05", "typeTextFieldData": "keyFD05", "typeKeywordFieldData": "textFD0OverTen5", "textDataFieldNoFields": "textFDNF05", "int0": 5} +{"index": {"_id":"7"}} +{"typeKeyword": "key06", "typeText": "text06", "typeKeywordFieldNoFieldData": "keyword06", "typeTextFieldData": "keyFD06OverTen", "typeKeywordFieldData": "textFD06", "textDataFieldNoFields": "textFDNF06", "int0": 6} +{"index": {"_id":"8"}} +{"typeKeyword": "key07", "typeText": "text07", "typeKeywordFieldNoFieldData": "keyword07", "typeTextFieldData": "keyFD07", "typeKeywordFieldData": "textFD07", "textDataFieldNoFields": "textFDNF07", "int0": 7}