diff --git a/integ-test/src/integration/resources/aws-logs/vpc_flow.sql b/integ-test/src/integration/resources/aws-logs/vpc_flow.sql index 77e27ab88..7c1f4084f 100644 --- a/integ-test/src/integration/resources/aws-logs/vpc_flow.sql +++ b/integ-test/src/integration/resources/aws-logs/vpc_flow.sql @@ -21,9 +21,9 @@ OPTIONS ( ); INSERT INTO {table_name} VALUES - (1, '123456789012', 'eni-abc123', '10.0.0.1', '10.0.0.2', 12345, 80, 6, 10, 200, 1622548800, 1622548860, 'ACCEPT', 'OK'), -- 5:00:00 AM to 5:01:00 AM PDT - (2, '123456789012', 'eni-def456', '10.0.0.1', '10.0.0.2', 12346, 443, 6, 5, 150, 1622548900, 1622548960, 'ACCEPT', 'OK'), -- 5:01:40 AM to 5:02:40 AM PDT - (3, '123456789013', 'eni-ghi789', '10.0.0.3', '10.0.0.4', 12347, 22, 6, 15, 300, 1622549400, 1622549460, 'ACCEPT', 'OK'), -- 5:10:00 AM to 5:11:00 AM PDT - (4, '123456789013', 'eni-jkl012', '10.0.0.5', '10.0.0.6', 12348, 21, 6, 20, 400, 1622549500, 1622549560, 'REJECT', 'OK'), -- 5:11:40 AM to 5:12:40 AM PDT - (5, '123456789014', 'eni-mno345', '10.0.0.7', '10.0.0.8', 12349, 25, 6, 25, 500, 1622550000, 1622550060, 'ACCEPT', 'OK') -- 5:20:00 AM to 5:21:00 AM PDT + (1, '123456789012', 'eni-abc123', '10.0.0.1', '10.0.0.2', 12345, 80, 6, 10, 200, 1698814800, 1698814860, 'ACCEPT', 'OK'), -- 05:00:00 to 05:01:00 UTC + (2, '123456789012', 'eni-def456', '10.0.0.1', '10.0.0.2', 12346, 443, 6, 5, 150, 1698814900, 1698814960, 'ACCEPT', 'OK'), -- 05:01:40 to 05:02:40 UTC + (3, '123456789013', 'eni-ghi789', '10.0.0.3', '10.0.0.4', 12347, 22, 6, 15, 300, 1698815400, 1698815460, 'ACCEPT', 'OK'), -- 05:10:00 to 05:11:00 UTC + (4, '123456789013', 'eni-jkl012', '10.0.0.5', '10.0.0.6', 12348, 21, 6, 20, 400, 1698815500, 1698815560, 'REJECT', 'OK'), -- 05:11:40 to 05:12:40 UTC + (5, '123456789014', 'eni-mno345', '10.0.0.7', '10.0.0.8', 12349, 25, 6, 25, 500, 1698816000, 1698816060, 'ACCEPT', 'OK') -- 05:20:00 to 05:21:00 UTC ; diff --git a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewIntegrationsITSuite.scala b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewIntegrationsITSuite.scala index 9333d1034..1c9e3a867 100644 --- a/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewIntegrationsITSuite.scala +++ b/integ-test/src/integration/scala/org/opensearch/flint/spark/FlintSparkMaterializedViewIntegrationsITSuite.scala @@ -7,6 +7,8 @@ package org.opensearch.flint.spark import java.io.File import java.sql.Timestamp +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter import org.opensearch.flint.spark.mv.FlintSparkMaterializedView.getFlintIndexName import org.scalatest.matchers.should.Matchers @@ -52,7 +54,7 @@ class FlintSparkMaterializedViewIntegrationsITSuite extends FlintSparkSuite with |""".stripMargin) .assertIndexData( Row( - Timestamp.valueOf("2021-06-01 05:00:00"), + timestampFromUTC("2023-11-01T05:00:00Z"), "ACCEPT", "10.0.0.1", "10.0.0.2", @@ -61,7 +63,7 @@ class FlintSparkMaterializedViewIntegrationsITSuite extends FlintSparkSuite with 350.0, 15), Row( - Timestamp.valueOf("2021-06-01 05:10:00"), + timestampFromUTC("2023-11-01T05:10:00Z"), "ACCEPT", "10.0.0.3", "10.0.0.4", @@ -70,7 +72,7 @@ class FlintSparkMaterializedViewIntegrationsITSuite extends FlintSparkSuite with 300.0, 15), Row( - Timestamp.valueOf("2021-06-01 05:10:00"), + timestampFromUTC("2023-11-01T05:10:00Z"), "REJECT", "10.0.0.5", "10.0.0.6", @@ -129,7 +131,7 @@ class FlintSparkMaterializedViewIntegrationsITSuite extends FlintSparkSuite with | eventCategory |""".stripMargin) .assertIndexData(Row( - Timestamp.valueOf("2023-10-31 22:00:00"), + timestampFromUTC("2023-11-01T05:00:00Z"), "IAMUser", "123456789012", "MyRole", @@ -211,4 +213,9 @@ class FlintSparkMaterializedViewIntegrationsITSuite extends FlintSparkSuite with checkAnswer(actualRows, expectedRows) } } + + private def timestampFromUTC(utcString: String): Timestamp = { + val instant = ZonedDateTime.parse(utcString, DateTimeFormatter.ISO_DATE_TIME).toInstant + Timestamp.from(instant) + } }