From 631502b2e4a56820aee5b2cf5a4c7952963e4a14 Mon Sep 17 00:00:00 2001 From: javsanbel2 Date: Thu, 21 Nov 2024 16:35:09 +0100 Subject: [PATCH] fix main problem with tests --- ...etadataSchedulerApiaryIntegrationTest.java | 37 ++++++++++++++++--- .../integration/utils/ContainerTestUtils.java | 2 +- pom.xml | 5 ++- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java index ccbf19c6..321a89ad 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java @@ -36,6 +36,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; @@ -45,6 +46,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; import org.testcontainers.containers.localstack.LocalStackContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @@ -54,6 +56,7 @@ import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import com.amazonaws.services.sqs.AmazonSQS; +import com.amazonaws.services.sqs.model.CreateQueueResult; import com.amazonaws.services.sqs.model.PurgeQueueRequest; import com.amazonaws.services.sqs.model.SendMessageRequest; @@ -64,19 +67,27 @@ import com.expediagroup.beekeeper.integration.model.AlterTableSqsMessage; import com.expediagroup.beekeeper.integration.model.CreateTableSqsMessage; import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils; +import com.expediagroup.beekeeper.integration.utils.HiveTestUtils; import com.expediagroup.beekeeper.scheduler.apiary.BeekeeperSchedulerApiary; +import com.hotels.beeju.extensions.ThriftHiveMetaStoreJUnitExtension; + @Testcontainers public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends BeekeeperIntegrationTestBase { private static final int TIMEOUT = 30; + private static final String DRY_RUN_ENABLED_PROPERTY = "properties.dry-run-enabled"; private static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url"; + private static final String METASTORE_URI_PROPERTY = "properties.metastore-uri"; private static final String QUEUE = "apiary-receiver-queue"; private static final String SCHEDULED_EXPIRED_METRIC = "metadata-scheduled"; private static final String HEALTHCHECK_URI = "http://localhost:8080/actuator/health"; private static final String PROMETHEUS_URI = "http://localhost:8080/actuator/prometheus"; + private static final String S3_ACCESS_KEY = "access"; + private static final String S3_SECRET_KEY = "secret"; + private static final String PARTITION_KEYS = "{ \"event_date\": \"date\", \"event_hour\": \"smallint\"}"; private static final String PARTITION_A_VALUES = "[ \"2020-01-01\", \"0\" ]"; private static final String PARTITION_B_VALUES = "[ \"2020-01-01\", \"1\" ]"; @@ -84,30 +95,44 @@ public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends Beek private static final String PARTITION_B_NAME = "event_date=2020-01-01/event_hour=1"; private static final String LOCATION_A = "s3://bucket/table1/partition"; private static final String LOCATION_B = "s3://bucket/table2/partition"; + private static final String TABLE_PATH = "/tmp/bucket/" + DATABASE_NAME_VALUE + "/" + TABLE_NAME_VALUE + "/"; @Container private static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS); private static AmazonSQS amazonSQS; + private static String queueUrl; + + @RegisterExtension + public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( + DATABASE_NAME_VALUE); + + private HiveTestUtils hiveTestUtils; + private HiveMetaStoreClient metastoreClient; @BeforeAll public static void init() { - String queueUrl = ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE); - System.setProperty(APIARY_QUEUE_URL_PROPERTY, queueUrl); - + System.setProperty(DRY_RUN_ENABLED_PROPERTY, "false"); amazonSQS = ContainerTestUtils.sqsClient(SQS_CONTAINER, AWS_REGION); - amazonSQS.createQueue(QUEUE); + CreateQueueResult queue = amazonSQS.createQueue(QUEUE); + queueUrl = queue.getQueueUrl(); + System.setProperty(APIARY_QUEUE_URL_PROPERTY, queueUrl); } @AfterAll public static void teardown() { System.clearProperty(APIARY_QUEUE_URL_PROPERTY); + System.clearProperty(DRY_RUN_ENABLED_PROPERTY); amazonSQS.shutdown(); } @BeforeEach public void setup() { - amazonSQS.purgeQueue(new PurgeQueueRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE))); + System.setProperty(METASTORE_URI_PROPERTY, thriftHiveMetaStore.getThriftConnectionUri()); + metastoreClient = thriftHiveMetaStore.client(); + hiveTestUtils = new HiveTestUtils(metastoreClient); + + amazonSQS.purgeQueue(new PurgeQueueRequest(queueUrl)); executorService.execute(() -> BeekeeperSchedulerApiary.main(new String[] {})); await().atMost(Duration.ONE_MINUTE).until(BeekeeperSchedulerApiary::isRunning); } @@ -230,7 +255,7 @@ public void prometheus() { } private SendMessageRequest sendMessageRequest(String payload) { - return new SendMessageRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE), payload); + return new SendMessageRequest(queueUrl, payload); } private void assertExpiredMetadata(HousekeepingMetadata actual, String expectedPath, String partitionName) { diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java index c8cce2fc..d0d96b7b 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java @@ -36,7 +36,7 @@ public static MySQLContainer mySqlContainer() { } public static LocalStackContainer awsContainer(LocalStackContainer.Service service) { - return new LocalStackContainer().withServices(service); + return new LocalStackContainer("0.13.1").withServices(service); } public static String awsServiceEndpoint(LocalStackContainer awsContainer, LocalStackContainer.Service service) { diff --git a/pom.xml b/pom.xml index 4683bea5..a2008896 100644 --- a/pom.xml +++ b/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 @@ -50,7 +51,7 @@ 1.27 2.7.9 5.3.25 - 1.17.1 + 1.17.6 11-slim