fix main problem with tests

ExpediaGroup · Nov 21, 2024 · 631502b · 631502b
1 parent 8b1ca85
commit 631502b
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 9 deletions.
diff --git a/...iagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/...iagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java
@@ -36,6 +36,7 @@
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
@@ -45,6 +46,7 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
 import org.testcontainers.containers.localstack.LocalStackContainer;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
@@ -54,6 +56,7 @@
 import io.micrometer.core.instrument.composite.CompositeMeterRegistry;
 
 import com.amazonaws.services.sqs.AmazonSQS;
+import com.amazonaws.services.sqs.model.CreateQueueResult;
 import com.amazonaws.services.sqs.model.PurgeQueueRequest;
 import com.amazonaws.services.sqs.model.SendMessageRequest;
 
@@ -64,50 +67,72 @@
 import com.expediagroup.beekeeper.integration.model.AlterTableSqsMessage;
 import com.expediagroup.beekeeper.integration.model.CreateTableSqsMessage;
 import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils;
+import com.expediagroup.beekeeper.integration.utils.HiveTestUtils;
 import com.expediagroup.beekeeper.scheduler.apiary.BeekeeperSchedulerApiary;
 
+import com.hotels.beeju.extensions.ThriftHiveMetaStoreJUnitExtension;
+
 @Testcontainers
 public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends BeekeeperIntegrationTestBase {
 
   private static final int TIMEOUT = 30;
+  private static final String DRY_RUN_ENABLED_PROPERTY = "properties.dry-run-enabled";
   private static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url";
+  private static final String METASTORE_URI_PROPERTY = "properties.metastore-uri";
 
   private static final String QUEUE = "apiary-receiver-queue";
   private static final String SCHEDULED_EXPIRED_METRIC = "metadata-scheduled";
   private static final String HEALTHCHECK_URI = "http://localhost:8080/actuator/health";
   private static final String PROMETHEUS_URI = "http://localhost:8080/actuator/prometheus";
 
+  private static final String S3_ACCESS_KEY = "access";
+  private static final String S3_SECRET_KEY = "secret";
+
   private static final String PARTITION_KEYS = "{ \"event_date\": \"date\", \"event_hour\": \"smallint\"}";
   private static final String PARTITION_A_VALUES = "[ \"2020-01-01\", \"0\" ]";
   private static final String PARTITION_B_VALUES = "[ \"2020-01-01\", \"1\" ]";
   private static final String PARTITION_A_NAME = "event_date=2020-01-01/event_hour=0";
   private static final String PARTITION_B_NAME = "event_date=2020-01-01/event_hour=1";
   private static final String LOCATION_A = "s3://bucket/table1/partition";
   private static final String LOCATION_B = "s3://bucket/table2/partition";
+  private static final String TABLE_PATH = "/tmp/bucket/" + DATABASE_NAME_VALUE + "/" + TABLE_NAME_VALUE + "/";
 
   @Container
   private static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS);
   private static AmazonSQS amazonSQS;
+  private static String queueUrl;
+
+  @RegisterExtension
+  public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension(
+      DATABASE_NAME_VALUE);
+
+  private HiveTestUtils hiveTestUtils;
+  private HiveMetaStoreClient metastoreClient;
 
   @BeforeAll
   public static void init() {
-    String queueUrl = ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE);
-    System.setProperty(APIARY_QUEUE_URL_PROPERTY, queueUrl);
-
+    System.setProperty(DRY_RUN_ENABLED_PROPERTY, "false");
     amazonSQS = ContainerTestUtils.sqsClient(SQS_CONTAINER, AWS_REGION);
-    amazonSQS.createQueue(QUEUE);
+    CreateQueueResult queue = amazonSQS.createQueue(QUEUE);
+    queueUrl = queue.getQueueUrl();
+    System.setProperty(APIARY_QUEUE_URL_PROPERTY, queueUrl);
   }
 
   @AfterAll
   public static void teardown() {
     System.clearProperty(APIARY_QUEUE_URL_PROPERTY);
+    System.clearProperty(DRY_RUN_ENABLED_PROPERTY);
 
     amazonSQS.shutdown();
   }
 
   @BeforeEach
   public void setup() {
-    amazonSQS.purgeQueue(new PurgeQueueRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE)));
+    System.setProperty(METASTORE_URI_PROPERTY, thriftHiveMetaStore.getThriftConnectionUri());
+    metastoreClient = thriftHiveMetaStore.client();
+    hiveTestUtils = new HiveTestUtils(metastoreClient);
+
+    amazonSQS.purgeQueue(new PurgeQueueRequest(queueUrl));
     executorService.execute(() -> BeekeeperSchedulerApiary.main(new String[] {}));
     await().atMost(Duration.ONE_MINUTE).until(BeekeeperSchedulerApiary::isRunning);
   }
@@ -230,7 +255,7 @@ public void prometheus() {
   }
 
   private SendMessageRequest sendMessageRequest(String payload) {
-    return new SendMessageRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE), payload);
+    return new SendMessageRequest(queueUrl, payload);
   }
 
   private void assertExpiredMetadata(HousekeepingMetadata actual, String expectedPath, String partitionName) {

diff --git a/...-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java b/...-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ContainerTestUtils.java
@@ -36,7 +36,7 @@ public static MySQLContainer mySqlContainer() {
   }
 
   public static LocalStackContainer awsContainer(LocalStackContainer.Service service) {
-    return new LocalStackContainer().withServices(service);
+    return new LocalStackContainer("0.13.1").withServices(service);
   }
 
   public static String awsServiceEndpoint(LocalStackContainer awsContainer, LocalStackContainer.Service service) {

diff --git a/pom.xml b/pom.xml
@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
@@ -50,7 +51,7 @@
     <snakeyaml.version>1.27</snakeyaml.version>
     <springframework.boot.version>2.7.9</springframework.boot.version>
     <springframework.version>5.3.25</springframework.version>
-    <testcontainers.version>1.17.1</testcontainers.version>
+    <testcontainers.version>1.17.6</testcontainers.version>
     <docker.from.tag>11-slim</docker.from.tag>
   </properties>