From 8c1ce389ca07991d05b312a55fec776e1889ec6d Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Sun, 24 Nov 2024 23:38:05 +0000 Subject: [PATCH] cleanup --- .../cleanup/validation/IcebergValidator.java | 12 +++++----- .../cleanup/aws/S3PathCleanerTest.java | 12 ---------- .../cleanup/hive/HiveMetadataCleanerTest.java | 5 ++--- .../validation/IcebergValidatorTest.java | 22 +++++++++++++++++++ 4 files changed, 29 insertions(+), 22 deletions(-) diff --git a/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/validation/IcebergValidator.java b/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/validation/IcebergValidator.java index f94a5d64..c4167232 100644 --- a/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/validation/IcebergValidator.java +++ b/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/validation/IcebergValidator.java @@ -37,9 +37,9 @@ public IcebergValidator(CleanerClientFactory cleanerClientFactory) { } /** - * Beekeeper does not support Iceberg format right now. Iceberg tables in Hive Metastore do not store partition information, - * so Beekeeper tries to clean up the entire table because that information is missing. This method checks if - * the table is an Iceberg table and throws IcebergTableFoundException to stop the process. + * Beekeeper currently does not support the Iceberg format. Iceberg tables in the Hive Metastore do not store partition information, + * causing Beekeeper to attempt to clean up the entire table due to the missing information. This method checks if + * the table is an Iceberg table and throws a BeekeeperIcebergException to stop the process. * * @param databaseName * @param tableName @@ -53,13 +53,11 @@ public void throwExceptionIfIceberg(String databaseName, String tableName) { if (tableType.contains("iceberg") || format.contains("iceberg") || (outputFormat != null && outputFormat.toLowerCase().contains("iceberg"))) { throw new BeekeeperIcebergException( - format("Iceberg table %s.%s is not currently supported in Beekeeper.", databaseName, - tableName)); + format("Iceberg table %s.%s is not currently supported in Beekeeper.", databaseName, tableName)); } } catch (Exception e) { throw new BeekeeperIcebergException( - format("Unexpected exception when identifying if table %s.%s is Iceberg.", databaseName, - tableName), e); + format("Unexpected exception when identifying if table %s.%s is Iceberg.", databaseName, tableName), e); } } } diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java index 7a046c98..35387a42 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/aws/S3PathCleanerTest.java @@ -383,18 +383,6 @@ void shouldThrowBeekeeperIcebergExceptionWhenIcebergTableDetected() { verifyNoInteractions(bytesDeletedReporter); } - @Test - void shouldNotReportBytesDeletedWhenIcebergValidatorThrows() { - doThrow(new BeekeeperIcebergException("Iceberg tables are not supported")) - .when(icebergValidator) - .throwExceptionIfIceberg(housekeepingPath.getDatabaseName(), housekeepingPath.getTableName()); - - assertThatExceptionOfType(BeekeeperIcebergException.class) - .isThrownBy(() -> s3PathCleaner.cleanupPath(housekeepingPath)); - - verify(bytesDeletedReporter, never()).reportTaggable(anyLong(), any(), any()); - } - @Test void shouldProceedWithDeletionWhenNotIcebergTable() { // setting up objects in the bucket diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/hive/HiveMetadataCleanerTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/hive/HiveMetadataCleanerTest.java index 3ed5aa8b..6a579cb8 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/hive/HiveMetadataCleanerTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/hive/HiveMetadataCleanerTest.java @@ -95,7 +95,7 @@ public void tableExists() { } @Test - public void dropTableWhenIcebergTable() { + public void doesNotDropTableWhenIcebergTable() { when(housekeepingMetadata.getDatabaseName()).thenReturn(DATABASE); when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); doThrow(new BeekeeperIcebergException("Iceberg table")) @@ -106,13 +106,12 @@ public void dropTableWhenIcebergTable() { () -> cleaner.dropTable(housekeepingMetadata, hiveClient) ); - // Verify that dropTable was not called on hiveClient verify(hiveClient, never()).dropTable(DATABASE, TABLE_NAME); verify(deletedMetadataReporter, never()).reportTaggable(housekeepingMetadata, MetadataType.HIVE_TABLE); } @Test - public void dropPartitionWhenIcebergTable() { + public void doesNotDropPartitionWhenIcebergTable() { when(housekeepingMetadata.getDatabaseName()).thenReturn(DATABASE); when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); doThrow(new BeekeeperIcebergException("Iceberg table")) diff --git a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/validation/IcebergValidatorTest.java b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/validation/IcebergValidatorTest.java index 663a40bb..dcaf308c 100644 --- a/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/validation/IcebergValidatorTest.java +++ b/beekeeper-cleanup/src/test/java/com/expediagroup/beekeeper/cleanup/validation/IcebergValidatorTest.java @@ -91,4 +91,26 @@ public void shouldThrowExceptionWhenOutputFormatContainsIceberg() throws Excepti icebergValidator.throwExceptionIfIceberg("db", "table"); } + + @Test(expected = BeekeeperIcebergException.class) + public void shouldThrowExceptionWhenFormatIsNullButTableTypeIsIceberg() throws Exception { + Map properties = new HashMap<>(); + properties.put("table_type", "ICEBERG"); + + when(cleanerClient.getTableProperties("db", "table")).thenReturn(properties); + when(cleanerClient.getOutputFormat("db", "table")).thenReturn(""); + + icebergValidator.throwExceptionIfIceberg("db", "table"); + } + + @Test + public void shouldNotThrowExceptionWhenOutputFormatIsNull() throws Exception { + Map properties = new HashMap<>(); + properties.put("table_type", "HIVE_TABLE"); + + when(cleanerClient.getTableProperties("db", "table")).thenReturn(properties); + when(cleanerClient.getOutputFormat("db", "table")).thenReturn(null); + + icebergValidator.throwExceptionIfIceberg("db", "table"); + } }