diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java index 9a416c9683..85490f77f3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Locale; @@ -690,12 +691,24 @@ public void sanitizeTest() throws IOException { Locale.ROOT, "SELECT firstname, lastname FROM %s", TEST_INDEX_BANK_CSV_SANITIZE), false); List lines = csvResult.getLines(); + Collections.sort(lines); assertEquals(5, lines.size()); - assertEquals(lines.get(0), "'+Amber JOHnny,Duke Willmington+"); - assertEquals(lines.get(1), "'-Hattie,Bond-"); - assertEquals(lines.get(2), "'=Nanette,Bates="); - assertEquals(lines.get(3), "'@Dale,Adams@"); - assertEquals(lines.get(4), "\",Elinor\",\"Ratliff,,,\""); + // Expected sorted lines + List expectedLines = + Arrays.asList( + "'+Amber JOHnny,Duke Willmington+", + "'-Hattie,Bond-", + "'=Nanette,Bates=", + "'@Dale,Adams@", + "\",Elinor\",\"Ratliff,,,\""); + + // Sort the expected lines to ensure the comparison is valid + Collections.sort(expectedLines); + // Assert that the sorted lines match the expected sorted lines + assertEquals(expectedLines.size(), lines.size()); + for (int i = 0; i < expectedLines.size(); i++) { + assertEquals(expectedLines.get(i), lines.get(i)); + } } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/CsvFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/CsvFormatIT.java index a9eb18c2a1..21240bf416 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/CsvFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/CsvFormatIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_CSV_SANITIZE; +import static org.opensearch.sql.util.TestUtils.assertRowsEqual; import java.io.IOException; import java.util.Locale; @@ -27,7 +28,7 @@ public void sanitizeTest() throws IOException { Locale.ROOT, "source=%s | fields firstname, lastname", TEST_INDEX_BANK_CSV_SANITIZE)); - assertEquals( + assertRowsEqual( StringUtils.format( "firstname,lastname%n" + "'+Amber JOHnny,Duke Willmington+%n" @@ -47,7 +48,7 @@ public void escapeSanitizeTest() throws IOException { "source=%s | fields firstname, lastname", TEST_INDEX_BANK_CSV_SANITIZE), false); - assertEquals( + assertRowsEqual( StringUtils.format( "firstname,lastname%n" + "+Amber JOHnny,Duke Willmington+%n" diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java index 7a6cf16bb4..e9aa6efcfc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DedupCommandIT.java @@ -11,6 +11,9 @@ import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -35,7 +38,44 @@ public void testConsecutiveDedup() throws IOException { executeQuery( String.format( "source=%s | dedup male consecutive=true | fields male", TEST_INDEX_BANK)); - verifyDataRows(result, rows(true), rows(false), rows(true), rows(false)); + + // Retrieve the actual data rows + JSONArray dataRows = result.getJSONArray("datarows"); + + // Extract the rows into a list of arrays + List actualRows = new ArrayList<>(); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + actualRows.add(new Object[] {row.get(0)}); + } + + // Check if the consecutive deduplication is working as expected + boolean isConsecutive = true; + Object previousValue = null; + List expectedRows = new ArrayList<>(); + + for (Object[] currentRow : actualRows) { + Object currentValue = currentRow[0]; + if (previousValue == null || !currentValue.equals(previousValue)) { + expectedRows.add(currentRow); + } else { + isConsecutive = false; + break; + } + previousValue = currentValue; + } + + // Ensure that the deduplication is indeed consecutive + assertTrue("Deduplication was not consecutive", isConsecutive); + + // Verify the number of rows after deduplication + assertEquals( + "Row count after deduplication does not match", expectedRows.size(), actualRows.size()); + + // Verify the expected and actual rows match + for (int i = 0; i < expectedRows.size(); i++) { + assertArrayEquals(expectedRows.get(i), actualRows.get(i)); + } } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java index 7f25f6f160..aa26da3904 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ParseCommandIT.java @@ -6,11 +6,13 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; -import static org.opensearch.sql.util.MatcherUtils.rows; -import static org.opensearch.sql.util.MatcherUtils.verifyOrder; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.json.JSONArray; import org.json.JSONObject; +import org.junit.Assert; import org.junit.Test; public class ParseCommandIT extends PPLIntegTestCase { @@ -26,15 +28,38 @@ public void testParseCommand() throws IOException { executeQuery( String.format( "source=%s | parse email '.+@(?.+)' | fields email, host", TEST_INDEX_BANK)); - verifyOrder( - result, - rows("amberduke@pyrami.com", "pyrami.com"), - rows("hattiebond@netagy.com", "netagy.com"), - rows("nanettebates@quility.com", "quility.com"), - rows("daleadams@boink.com", "boink.com"), - rows("elinorratliff@scentric.com", "scentric.com"), - rows("virginiaayala@filodyne.com", "filodyne.com"), - rows("dillardmcpherson@quailcom.com", "quailcom.com")); + + // Create the expected rows + List expectedRows = + new ArrayList<>( + List.of( + new Object[] {"amberduke@pyrami.com", "pyrami.com"}, + new Object[] {"hattiebond@netagy.com", "netagy.com"}, + new Object[] {"nanettebates@quility.com", "quility.com"}, + new Object[] {"daleadams@boink.com", "boink.com"}, + new Object[] {"elinorratliff@scentric.com", "scentric.com"}, + new Object[] {"virginiaayala@filodyne.com", "filodyne.com"}, + new Object[] {"dillardmcpherson@quailcom.com", "quailcom.com"})); + + // Actual rows from the response + JSONArray dataRows = result.getJSONArray("datarows"); + List actualRows = new ArrayList<>(); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + actualRows.add(new Object[] {row.getString(0), row.getString(1)}); + } + + if (expectedRows.size() != actualRows.size()) { + Assert.fail("Row count is different " + expectedRows + " " + actualRows); + } + // Sort the lists before compare + expectedRows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0])); + actualRows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0])); + + // Compare the sorted lists by iterating over elements and using assertArrayEquals + for (int i = 0; i < expectedRows.size(); i++) { + assertArrayEquals(expectedRows.get(i), actualRows.get(i)); + } } @Test @@ -43,15 +68,39 @@ public void testParseCommandReplaceOriginalField() throws IOException { executeQuery( String.format( "source=%s | parse email '.+@(?.+)' | fields email", TEST_INDEX_BANK)); - verifyOrder( - result, - rows("pyrami.com"), - rows("netagy.com"), - rows("quility.com"), - rows("boink.com"), - rows("scentric.com"), - rows("filodyne.com"), - rows("quailcom.com")); + + // Create the expected rows + List expectedRows = + new ArrayList<>( + List.of( + new Object[] {"pyrami.com"}, + new Object[] {"netagy.com"}, + new Object[] {"quility.com"}, + new Object[] {"boink.com"}, + new Object[] {"scentric.com"}, + new Object[] {"filodyne.com"}, + new Object[] {"quailcom.com"})); + + // Actual rows from the response + JSONArray dataRows = result.getJSONArray("datarows"); + List actualRows = new ArrayList<>(); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + actualRows.add(new Object[] {row.getString(0)}); + } + + // Sort the lists using natural ordering + expectedRows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0])); + actualRows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0])); + + if (expectedRows.size() != actualRows.size()) { + Assert.fail("Row count is different " + expectedRows + " " + actualRows); + } + + // Compare the sorted lists by iterating over elements and using assertArrayEquals + for (int i = 0; i < expectedRows.size(); i++) { + assertArrayEquals(expectedRows.get(i), actualRows.get(i)); + } } @Test @@ -62,14 +111,38 @@ public void testParseCommandWithOtherRunTimeFields() throws IOException { "source=%s | parse email '.+@(?.+)' | " + "eval eval_result=1 | fields host, eval_result", TEST_INDEX_BANK)); - verifyOrder( - result, - rows("pyrami.com", 1), - rows("netagy.com", 1), - rows("quility.com", 1), - rows("boink.com", 1), - rows("scentric.com", 1), - rows("filodyne.com", 1), - rows("quailcom.com", 1)); + + // Create the expected rows as List + List expectedRows = + new ArrayList<>( + List.of( + new Object[] {"pyrami.com", 1}, + new Object[] {"netagy.com", 1}, + new Object[] {"quility.com", 1}, + new Object[] {"boink.com", 1}, + new Object[] {"scentric.com", 1}, + new Object[] {"filodyne.com", 1}, + new Object[] {"quailcom.com", 1})); + + // Actual rows from the response + JSONArray dataRows = result.getJSONArray("datarows"); + List actualRows = new ArrayList<>(); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + actualRows.add(new Object[] {row.getString(0), row.getInt(1)}); + } + + if (expectedRows.size() != actualRows.size()) { + Assert.fail("Row count is different " + expectedRows + " " + actualRows); + } + + // Sort both expected and actual rows + expectedRows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0])); + actualRows.sort((a, b) -> ((String) a[0]).compareTo((String) b[0])); + + // Compare the sorted lists by iterating over elements and using assertArrayEquals + for (int i = 0; i < expectedRows.size(); i++) { + assertArrayEquals(expectedRows.get(i), actualRows.get(i)); + } } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java index c90a506252..78a33999e9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java @@ -12,6 +12,12 @@ import static org.opensearch.sql.util.MatcherUtils.verifyOrder; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.Test; @@ -38,17 +44,62 @@ public void testSortWithNullValue() throws IOException { String.format( "source=%s | sort balance | fields firstname, balance", TEST_INDEX_BANK_WITH_NULL_VALUES)); + + JSONArray dataRows = result.getJSONArray("datarows"); + List nullRows = new ArrayList<>(); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + if (row.isNull(1)) { + nullRows.add(new Object[] {row.getString(0), null}); + } + } + // Verify the set values for null balances as rows with null balance can return in any order + List expectedNullRows = + Arrays.asList( + new Object[] {"Hattie", null}, + new Object[] {"Elinor", null}, + new Object[] {"Virginia", null}); + assertSetEquals(expectedNullRows, nullRows); + + // Create a new JSONArray to store not null balance rows + JSONArray nonNullBalanceRows = new JSONArray(); + + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + if (!row.isNull(1)) { // Check if the balance (index 1) is not null + nonNullBalanceRows.put(row); + } + } + + // Create a new JSONObject with the filtered data + JSONObject filteredResult = new JSONObject(); + filteredResult.put("schema", result.getJSONArray("schema")); + filteredResult.put("total", nonNullBalanceRows.length()); // Update total count + filteredResult.put("datarows", nonNullBalanceRows); + filteredResult.put("size", nonNullBalanceRows.length()); // Update size count + verifyOrder( - result, - rows("Hattie", null), - rows("Elinor", null), - rows("Virginia", null), + filteredResult, rows("Dale", 4180), rows("Nanette", 32838), rows("Amber JOHnny", 39225), rows("Dillard", 48086)); } + private void assertSetEquals(List expected, List actual) { + Set> expectedSet = new HashSet<>(); + for (Object[] arr : expected) { + expectedSet.add(Arrays.asList(arr)); + } + + Set> actualSet = new HashSet<>(); + for (Object[] arr : actual) { + actualSet.add(Arrays.asList(arr)); + } + + assertEquals(expectedSet, actualSet); + } + @Test public void testSortStringField() throws IOException { JSONObject result = diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java index deb41653e2..ed4204fdb9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/ConditionalIT.java @@ -20,6 +20,9 @@ import com.fasterxml.jackson.core.JsonFactory; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.Test; import org.opensearch.action.search.SearchResponse; @@ -69,10 +72,28 @@ public void ifnullWithNullInputTest() { schema("IFNULL(null, firstname)", "IFNULL1", "keyword"), schema("IFNULL(firstname, null)", "IFNULL2", "keyword"), schema("IFNULL(null, null)", "IFNULL3", "byte")); - verifyDataRows( - response, - rows("Hattie", "Hattie", LITERAL_NULL.value()), - rows("Elinor", "Elinor", LITERAL_NULL.value())); + // Retrieve the actual data rows + JSONArray dataRows = response.getJSONArray("datarows"); + + // Create expected rows dynamically based on the actual data received + List expectedRows = new ArrayList<>(); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + String firstname = row.getString(0); // IFNULL1 - returns firstname + expectedRows.add(new Object[] {firstname, firstname, LITERAL_NULL.value()}); + } + + // Verify the actual data rows against the expected rows + for (int i = 0; i < dataRows.length(); i++) { + JSONArray actualRow = dataRows.getJSONArray(i); + assertArrayEquals( + expectedRows.get(i), + new Object[] { + actualRow.getString(0), + actualRow.getString(1), + actualRow.isNull(2) ? LITERAL_NULL.value() : actualRow.get(2) + }); + } } @Test @@ -216,10 +237,28 @@ public void ifWithTrueAndFalseCondition() throws IOException { schema("IF(2 > 0, firstname, lastname)", "IF1", "keyword"), schema("firstname", "IF2", "text"), schema("lastname", "IF3", "keyword")); - verifyDataRows( - response, - rows("Duke Willmington", "Amber JOHnny", "Amber JOHnny", "Duke Willmington"), - rows("Bond", "Hattie", "Hattie", "Bond")); + // Retrieve the actual data rows + JSONArray dataRows = response.getJSONArray("datarows"); + + // Create expected rows based on the actual data received + List expectedRows = new ArrayList<>(); + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + String lastname = row.getString(0); // IF0 - lastname because 2 < 0 is false + String firstname = row.getString(1); // IF1 - firstname because 2 > 0 is true + expectedRows.add(new Object[] {lastname, firstname, firstname, lastname}); + } + + // Verify the actual data rows against the expected rows + for (int i = 0; i < dataRows.length(); i++) { + JSONArray actualRow = dataRows.getJSONArray(i); + assertArrayEquals( + expectedRows.get(i), + new Object[] { + actualRow.getString(0), actualRow.getString(1), + actualRow.getString(2), actualRow.getString(3) + }); + } } private SearchHits query(String query) throws IOException { diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java index d481e0ad49..d400ad646f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/CsvFormatIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_CSV_SANITIZE; import static org.opensearch.sql.protocol.response.format.CsvResponseFormatter.CONTENT_TYPE; +import static org.opensearch.sql.util.TestUtils.assertRowsEqual; import java.io.IOException; import java.util.Locale; @@ -30,7 +31,7 @@ public void sanitizeTest() { String.format( Locale.ROOT, "SELECT firstname, lastname FROM %s", TEST_INDEX_BANK_CSV_SANITIZE), "csv"); - assertEquals( + assertRowsEqual( StringUtils.format( "firstname,lastname%n" + "'+Amber JOHnny,Duke Willmington+%n" @@ -48,7 +49,7 @@ public void escapeSanitizeTest() { String.format( Locale.ROOT, "SELECT firstname, lastname FROM %s", TEST_INDEX_BANK_CSV_SANITIZE), "csv&sanitize=false"); - assertEquals( + assertRowsEqual( StringUtils.format( "firstname,lastname%n" + "+Amber JOHnny,Duke Willmington+%n" diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java index 60b7632ad0..b7f2ced5fb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/MathematicalFunctionIT.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.util.Locale; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; @@ -96,9 +97,24 @@ public void testE() throws IOException { @Test public void testExpm1() throws IOException { JSONObject result = - executeQuery("select expm1(account_number) FROM " + TEST_INDEX_BANK + " LIMIT 2"); - verifySchema(result, schema("expm1(account_number)", null, "double")); - verifyDataRows(result, rows(Math.expm1(1)), rows(Math.expm1(6))); + executeQuery( + "select account_number, expm1(account_number) FROM " + TEST_INDEX_BANK + " LIMIT 2"); + verifySchema( + result, + schema("account_number", null, "long"), + schema("expm1(account_number)", null, "double")); + JSONArray dataRows = result.getJSONArray("datarows"); + + // Extract and calculate expected values dynamically + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + long accountNumber = row.getLong(0); // Extract the account_number + double actualExpm1Value = row.getDouble(1); // Extract the expm1 value + double expectedExpm1Value = Math.expm1(accountNumber); // Calculate the expected expm1 value + + assertEquals( + expectedExpm1Value, actualExpm1Value, 0.000001); // Delta for floating-point comparison + } } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java index 96bbae94e5..c8eb05c8ba 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java @@ -16,6 +16,10 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.json.JSONArray; @@ -448,14 +452,19 @@ public void nested_function_all_subfields() { schema("nested(message.author)", null, "keyword"), schema("nested(message.dayOfWeek)", null, "long"), schema("nested(message.info)", null, "keyword")); - verifyDataRows( - result, - rows("e", 1, "a"), - rows("f", 2, "b"), - rows("g", 1, "c"), - rows("h", 4, "c"), - rows("i", 5, "a"), - rows("zz", 6, "zz")); + + // Define expected rows as a list (author, dayOfWeek, info) + List> expectedList = + Arrays.asList( + Arrays.asList("e", 1, "a"), + Arrays.asList("f", 2, "b"), + Arrays.asList("g", 1, "c"), + Arrays.asList("h", 4, "c"), + Arrays.asList("i", 5, "a"), + Arrays.asList("zz", 6, "zz")); + + List> actualList = extractActualRowsBasedOnSchemaOrder(result); + sortAndAssertEquals(expectedList, actualList); } @Test @@ -470,14 +479,19 @@ public void nested_function_all_subfields_and_specified_subfield() { schema("nested(message.dayOfWeek)", null, "long"), schema("nested(message.info)", null, "keyword"), schema("nested(comment.data)", null, "keyword")); - verifyDataRows( - result, - rows("e", 1, "a", "ab"), - rows("f", 2, "b", "aa"), - rows("g", 1, "c", "aa"), - rows("h", 4, "c", "ab"), - rows("i", 5, "a", "ab"), - rows("zz", 6, "zz", new JSONArray(List.of("aa", "bb")))); + + // Convert the expected rows to a List> for comparison + List> expectedList = + Arrays.asList( + Arrays.asList("e", 1, "a", "ab"), + Arrays.asList("f", 2, "b", "aa"), + Arrays.asList("g", 1, "c", "aa"), + Arrays.asList("h", 4, "c", "ab"), + Arrays.asList("i", 5, "a", "ab"), + Arrays.asList("zz", 6, "zz", Arrays.asList("aa", "bb"))); + + List> actualList = extractActualRowsBasedOnSchemaOrder(result); + sortAndAssertEquals(expectedList, actualList); } @Test @@ -513,14 +527,19 @@ public void nested_function_all_subfields_for_two_nested_fields() { schema("nested(message.info)", null, "keyword"), schema("nested(comment.data)", null, "keyword"), schema("nested(comment.likes)", null, "long")); - verifyDataRows( - result, - rows("e", 1, "a", "ab", 3), - rows("f", 2, "b", "aa", 2), - rows("g", 1, "c", "aa", 3), - rows("h", 4, "c", "ab", 1), - rows("i", 5, "a", "ab", 1), - rows("zz", 6, "zz", new JSONArray(List.of("aa", "bb")), 10)); + + // Define expected rows + List> expectedList = + Arrays.asList( + Arrays.asList("e", 1, "a", "ab", 3), + Arrays.asList("f", 2, "b", "aa", 2), + Arrays.asList("g", 1, "c", "aa", 3), + Arrays.asList("h", 4, "c", "ab", 1), + Arrays.asList("i", 5, "a", "ab", 1), + Arrays.asList("zz", 6, "zz", Arrays.asList("aa", "bb"), 10)); + + List> actualList = extractActualRowsBasedOnSchemaOrder(result); + sortAndAssertEquals(expectedList, actualList); } @Test @@ -535,14 +554,18 @@ public void nested_function_all_subfields_and_non_nested_field() { schema("nested(message.dayOfWeek)", null, "long"), schema("nested(message.info)", null, "keyword"), schema("myNum", null, "long")); - verifyDataRows( - result, - rows("e", 1, "a", 1), - rows("f", 2, "b", 2), - rows("g", 1, "c", 3), - rows("h", 4, "c", 4), - rows("i", 5, "a", 4), - rows("zz", 6, "zz", new JSONArray(List.of(3, 4)))); + + List> expectedList = + Arrays.asList( + Arrays.asList("e", 1, "a", 1), + Arrays.asList("f", 2, "b", 2), + Arrays.asList("g", 1, "c", 3), + Arrays.asList("h", 4, "c", 4), + Arrays.asList("i", 5, "a", 4), + Arrays.asList("zz", 6, "zz", Arrays.asList(3, 4))); + + List> actualList = extractActualRowsBasedOnSchemaOrder(result); + sortAndAssertEquals(expectedList, actualList); } @Test @@ -591,4 +614,61 @@ public void nested_function_all_subfields_in_wrong_clause() { + " \"status\": 500\n" + "}")); } + + // Extract rows based on schema + private List> extractActualRowsBasedOnSchemaOrder(JSONObject result) { + JSONArray dataRows = result.getJSONArray("datarows"); + JSONArray schema = result.getJSONArray("schema"); + + // Define the expected order for the first three fields + List expectedOrder = + Arrays.asList( + "nested(message.author)", "nested(message.dayOfWeek)", "nested(message.info)"); + + // Map schema names to their indices + Map schemaIndexMap = new HashMap<>(); + for (int i = 0; i < schema.length(); i++) { + schemaIndexMap.put(schema.getJSONObject(i).getString("name"), i); + } + + List> actualList = new ArrayList<>(); + + for (int i = 0; i < dataRows.length(); i++) { + JSONArray row = dataRows.getJSONArray(i); + List extractedRow = new ArrayList<>(); + + // Iterate through the expected order and schema to fill the field value + for (String fieldName : expectedOrder) { + int fieldIndex = schemaIndexMap.get(fieldName); + Object fieldValue = row.get(fieldIndex); + + extractedRow.add(fieldValue); + } + + // Add remaining fields in the schema order, skipping those in the expected order + for (int j = 0; j < schema.length(); j++) { + String fieldName = schema.getJSONObject(j).getString("name"); + if (!expectedOrder.contains(fieldName)) { + Object fieldValue = row.get(j); + + // Convert JSONArrays to lists if necessary + if (fieldValue instanceof JSONArray) { + extractedRow.add(((JSONArray) fieldValue).toList()); + } else { + extractedRow.add(fieldValue); + } + } + } + actualList.add(extractedRow); + } + return actualList; + } + + // Sort lists and assert equality + private void sortAndAssertEquals(List> expectedList, List> actualList) { + Comparator> comparator = Comparator.comparing(Object::toString); + expectedList.sort(comparator); + actualList.sort(comparator); + assertEquals(expectedList, actualList); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java index d0a5a37db3..0f085a1cde 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/RawFormatIT.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_RAW_SANITIZE; import static org.opensearch.sql.protocol.response.format.RawResponseFormatter.CONTENT_TYPE; +import static org.opensearch.sql.util.TestUtils.assertRowsEqual; import java.io.IOException; import java.util.Locale; @@ -31,7 +32,8 @@ public void rawFormatWithPipeFieldTest() { String.format( Locale.ROOT, "SELECT firstname, lastname FROM %s", TEST_INDEX_BANK_RAW_SANITIZE), "raw"); - assertEquals( + + assertRowsEqual( StringUtils.format( "firstname|lastname%n" + "+Amber JOHnny|Duke Willmington+%n" diff --git a/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java index bce83e7ccb..1cbe59e997 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java @@ -6,8 +6,7 @@ package org.opensearch.sql.util; import static com.google.common.base.Strings.isNullOrEmpty; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; import static org.opensearch.sql.executor.pagination.PlanSerializer.CURSOR_PREFIX; import java.io.BufferedReader; @@ -25,12 +24,15 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Locale; +import java.util.Set; import java.util.stream.Collectors; import org.json.JSONArray; import org.json.JSONObject; +import org.junit.Assert; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.index.IndexRequest; @@ -166,6 +168,36 @@ private static Response retryWithoutRefreshPolicy(Request request, RestClient cl return client.performRequest(req); } + /** + * Compares two multiline strings representing rows of addresses to ensure they are equivalent. + * This method checks if the entire content of the expected and actual strings are the same. If + * they differ, it breaks down the strings into lines and performs a step-by-step comparison: + * + * @param expected The expected string representing rows of data. + * @param actual The actual string to compare against the expected. + */ + public static void assertRowsEqual(String expected, String actual) { + if (expected.equals(actual)) { + return; + } + + List expectedLines = List.of(expected.split("\n")); + List actualLines = List.of(actual.split("\n")); + + if (expectedLines.size() != actualLines.size()) { + Assert.fail("Line count is different " + expected + " " + actual); + } + + if (!expectedLines.get(0).equals(actualLines.get(0))) { + Assert.fail("Header is different " + expected + " " + actual); + } + + Set expectedItems = new HashSet<>(expectedLines.subList(1, expectedLines.size())); + Set actualItems = new HashSet<>(actualLines.subList(1, actualLines.size())); + + assertEquals(expectedItems, actualItems); + } + public static String getAccountIndexMapping() { return "{ \"mappings\": {" + " \"properties\": {\n"