cleanup

beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java

@@ -21,10 +21,8 @@
 import static org.assertj.core.api.Assertions.assertThatCode;
 import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
 import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoInteractions;
 import static org.mockito.Mockito.when;
@@ -385,17 +383,14 @@ void shouldThrowBeekeeperIcebergExceptionWhenIcebergTableDetected() {
 
   @Test
   void shouldProceedWithDeletionWhenNotIcebergTable() {
-    // setting up objects in the bucket
-    amazonS3.putObject(bucket, key1, content); // Add the files
+    amazonS3.putObject(bucket, key1, content);
     amazonS3.putObject(bucket, key2, content);
 
-    // housekeepingPath is  set
     housekeepingPath.setPath("s3://" + bucket + "/" + keyRoot);
 
     assertThatCode(() -> s3PathCleaner.cleanupPath(housekeepingPath))
         .doesNotThrowAnyException();
 
-    // verify objects are deleted and reporter is called
     assertThat(amazonS3.doesObjectExist(bucket, key1)).isFalse();
     assertThat(amazonS3.doesObjectExist(bucket, key2)).isFalse();
 

beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/hive/HiveMetadataCleanerTest.java

@@ -57,7 +57,7 @@ public void typicalDropTable() {
     when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME);
 
     cleaner.dropTable(housekeepingMetadata, hiveClient);
-    // Verify that the IcebergValidator is called
+
     verify(icebergValidator).throwExceptionIfIceberg(DATABASE, TABLE_NAME);
     verify(hiveClient).dropTable(DATABASE, TABLE_NAME);
     verify(deletedMetadataReporter).reportTaggable(housekeepingMetadata, MetadataType.HIVE_TABLE);
@@ -77,6 +77,7 @@ public void typicalDropPartition() {
     verify(deletedMetadataReporter).reportTaggable(housekeepingMetadata, MetadataType.HIVE_PARTITION);
   }
 
+
   @Test
   public void dontReportWhenPartitionNotDropped() {
     when(housekeepingMetadata.getDatabaseName()).thenReturn(DATABASE);

beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/error/BeekeeperIcebergException.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2019 Expedia, Inc.
+ * Copyright (C) 2019-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperDryRunMetadataCleanupIntegrationTest.java

@@ -22,12 +22,14 @@
 import static org.awaitility.Awaitility.await;
 import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3;
 
+import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SKIPPED;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.LONG_CLEANUP_DELAY_VALUE;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE;
 
 import java.sql.SQLException;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -60,6 +62,7 @@
 
 import com.expediagroup.beekeeper.cleanup.monitoring.BytesDeletedReporter;
 import com.expediagroup.beekeeper.cleanup.monitoring.DeletedMetadataReporter;
+import com.expediagroup.beekeeper.core.model.HousekeepingStatus;
 import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils;
 import com.expediagroup.beekeeper.integration.utils.HiveTestUtils;
 import com.expediagroup.beekeeper.integration.utils.TestAppender;
@@ -94,6 +97,7 @@ public class BeekeeperDryRunMetadataCleanupIntegrationTest extends BeekeeperInte
 
   private static final String ROOT_PATH = "s3a://" + BUCKET + "/" + DATABASE_NAME_VALUE + "/";
 
+  private static final String TABLE_PATH = "s3a://" + BUCKET + "/" + DATABASE_NAME_VALUE + "/" + TABLE_NAME_VALUE + "/";
   private static final String PARTITIONED_TABLE_PATH = ROOT_PATH + PARTITIONED_TABLE_NAME + "/id1";
   private static final String PARTITION_ROOT_PATH = ROOT_PATH + "some_location/id1";
   private static final String PARTITION_PATH = PARTITION_ROOT_PATH + "/" + PARTITION_NAME + "/file1";

beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java

@@ -69,6 +69,8 @@
 import com.google.common.collect.ImmutableMap;
 
 import com.expediagroup.beekeeper.cleanup.monitoring.BytesDeletedReporter;
+import com.expediagroup.beekeeper.core.model.HousekeepingPath;
+import com.expediagroup.beekeeper.core.model.HousekeepingStatus;
 import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils;
 import com.expediagroup.beekeeper.integration.utils.HiveTestUtils;
 import com.expediagroup.beekeeper.metadata.cleanup.BeekeeperMetadataCleanup;
@@ -229,26 +231,23 @@ public void cleanupPartitionedTable() throws Exception {
 
   @Test
   public void shouldSkipCleanupForIcebergTable() throws Exception {
-    // Define custom table props and outputFormat for an Iceberg table
     Map<String, String> tableProperties = new HashMap<>();
     tableProperties.put("table_type", "ICEBERG");
     tableProperties.put("format", "ICEBERG/PARQUET");
     String outputFormat = "org.apache.iceberg.mr.hive.HiveIcebergOutputFormat";
-    // Create the Iceberg table in the Hive metastore
+
     hiveTestUtils.createTableWithProperties(
         PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true, tableProperties, outputFormat, true);
-    // Add data to the S3 bucket
     amazonS3.putObject(BUCKET, PARTITIONED_TABLE_OBJECT_KEY, TABLE_DATA);
-    // Insert expired metadata for the Iceberg table
+
     insertExpiredMetadata(PARTITIONED_TABLE_PATH, null);
-    // wait for cleanup process to run
+
     await()
         .atMost(TIMEOUT, TimeUnit.SECONDS)
         .until(() -> getExpiredMetadata().get(0).getHousekeepingStatus() == SKIPPED);
-    // Verify that the table still exists
-    assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isTrue(); // this is fine, the table is not changed
-    // Verify that the data in S3 is still present
-    assertThat(amazonS3.doesObjectExist(BUCKET, PARTITIONED_TABLE_OBJECT_KEY)).isTrue(); // this too is fine, the data is not changed
+
+    assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isTrue();
+    assertThat(amazonS3.doesObjectExist(BUCKET, PARTITIONED_TABLE_OBJECT_KEY)).isTrue();
   }
 
   @Test

beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java

@@ -33,7 +33,6 @@
 import com.expediagroup.beekeeper.cleanup.metadata.MetadataCleaner;
 import com.expediagroup.beekeeper.cleanup.path.PathCleaner;
 import com.expediagroup.beekeeper.core.error.BeekeeperException;
-import com.expediagroup.beekeeper.core.error.BeekeeperIcebergException;
 import com.expediagroup.beekeeper.core.model.HousekeepingMetadata;
 import com.expediagroup.beekeeper.core.model.HousekeepingStatus;
 import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository;

beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java

@@ -76,9 +76,8 @@ private boolean cleanUpPath(HousekeepingPath housekeepingPath) {
       log.warn("Will not clean up path \"{}\" because it is not valid.", housekeepingPath.getPath());
       return false;
     } catch (BeekeeperException e) {
-      // Handle Iceberg table by updating status to SKIPPED
       updateStatus(housekeepingPath, HousekeepingStatus.SKIPPED);
-      log.warn("Skipping cleanup for Iceberg table \"{}.{}\": {}", housekeepingPath.getDatabaseName(),
+      log.warn("Skipping cleanup for table \"{}.{}\": {}", housekeepingPath.getDatabaseName(),
           housekeepingPath.getTableName(), e.getMessage());
       return false;
     }

beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/service/SchedulerApiary.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2019-2020 Expedia, Inc.
+ * Copyright (C) 2019-2024 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/service/SchedulerApiaryTest.java

@@ -24,7 +24,6 @@
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoInteractions;