Simplify event details

ExpediaGroup · Nov 25, 2024 · ea6d5a9 · ea6d5a9
1 parent 3665e28
commit ea6d5a9
Show file tree

Hide file tree

Showing 8 changed files with 89 additions and 84 deletions.
diff --git a/...r-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java b/...r-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java
@@ -25,7 +25,7 @@ public BeekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryReposi
   }
 
   public void saveHistory(HousekeepingEntity housekeepingEntity, String status) {
-    String eventDetails = createEventDetails(housekeepingEntity);
+    String eventDetails = getEventDetails(housekeepingEntity);
 
     BeekeeperHistory event = BeekeeperHistory.builder()
         .id(housekeepingEntity.getId())
@@ -41,16 +41,7 @@ public void saveHistory(HousekeepingEntity housekeepingEntity, String status) {
     beekeeperHistoryRepository.save(event);
   }
 
-  private String createEventDetails(HousekeepingEntity entity) {
-    ObjectMapper mapper = new ObjectMapper();
-    mapper.findAndRegisterModules();
-    mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
-    mapper.registerModule(new JavaTimeModule());
-    try {
-      return mapper.writeValueAsString(entity);
-    } catch (JsonProcessingException e) {
-      log.warn("Error encountered writing object as string", e);
-    }
-    return "{}";
+  private String getEventDetails(HousekeepingEntity entity) {
+    return entity.toString();
   }
 }
diff --git a/.../test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java b/.../test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java
@@ -22,11 +22,6 @@
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 import org.springframework.test.context.support.AnnotationConfigContextLoader;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-
 import com.expediagroup.beekeeper.core.TestApplication;
 import com.expediagroup.beekeeper.core.model.HousekeepingMetadata;
 import com.expediagroup.beekeeper.core.model.HousekeepingPath;
@@ -53,25 +48,19 @@ public class BeekeeperHistoryRepositoryTest {
 
   private static final int PAGE = 0;
   private static final int PAGE_SIZE = 500;
-  protected ObjectMapper mapper;
 
   @Autowired
   private BeekeeperHistoryRepository repository;
 
   @BeforeEach
   public void setupDb() {
     repository.deleteAll();
-
-    mapper = new ObjectMapper();
-    mapper.findAndRegisterModules();
-    mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
-    mapper.registerModule(new JavaTimeModule());
   }
 
   @Test
-  public void typicalSave() throws JsonProcessingException {
-    BeekeeperHistory expiredEntry = createExpiredEventDetails(SCHEDULED);
-    BeekeeperHistory unreferencedEntry = createUnreferencedEventDetails(SCHEDULED);
+  public void typicalSave() {
+    BeekeeperHistory expiredEntry = createExpiredEvent(SCHEDULED);
+    BeekeeperHistory unreferencedEntry = createUnreferencedEvent(SCHEDULED);
 
     repository.save(expiredEntry);
     repository.save(unreferencedEntry);
@@ -86,10 +75,10 @@ public void typicalSave() throws JsonProcessingException {
   }
 
   @Test
-  public void expired_multipleStatuses() throws JsonProcessingException {
-    BeekeeperHistory scheduledEntry = createExpiredEventDetails(SCHEDULED);
-    BeekeeperHistory deletedEntry = createExpiredEventDetails(DELETED);
-    BeekeeperHistory failedEntry = createExpiredEventDetails(FAILED);
+  public void expired_multipleStatuses() {
+    BeekeeperHistory scheduledEntry = createExpiredEvent(SCHEDULED);
+    BeekeeperHistory deletedEntry = createExpiredEvent(DELETED);
+    BeekeeperHistory failedEntry = createExpiredEvent(FAILED);
 
     repository.save(scheduledEntry);
     repository.save(deletedEntry);
@@ -101,10 +90,10 @@ public void expired_multipleStatuses() throws JsonProcessingException {
   }
 
   @Test
-  public void unreferenced_multipleStatuses() throws JsonProcessingException {
-    BeekeeperHistory scheduledEntry = createUnreferencedEventDetails(SCHEDULED);
-    BeekeeperHistory deletedEntry = createUnreferencedEventDetails(DELETED);
-    BeekeeperHistory failedEntry = createUnreferencedEventDetails(FAILED);
+  public void unreferenced_multipleStatuses() {
+    BeekeeperHistory scheduledEntry = createUnreferencedEvent(SCHEDULED);
+    BeekeeperHistory deletedEntry = createUnreferencedEvent(DELETED);
+    BeekeeperHistory failedEntry = createUnreferencedEvent(FAILED);
 
     repository.save(scheduledEntry);
     repository.save(deletedEntry);
@@ -115,7 +104,7 @@ public void unreferenced_multipleStatuses() throws JsonProcessingException {
     assertThat(historyList.size()).isEqualTo(3);
   }
 
-  protected BeekeeperHistory createExpiredEventDetails(HousekeepingStatus status) throws JsonProcessingException {
+  protected BeekeeperHistory createExpiredEvent(HousekeepingStatus status) {
     HousekeepingMetadata entity = HousekeepingMetadata.builder()
         .cleanupAttempts(3)
         .cleanupDelay(PeriodDuration.parse("P1D"))
@@ -124,20 +113,18 @@ protected BeekeeperHistory createExpiredEventDetails(HousekeepingStatus status)
         .modifiedTimestamp(COLUMN_TIMESTAMP)
         .build();
 
-    String stringDetails = mapper.writeValueAsString(entity);
-    return createHistoryEntry("EXPIRED", status, stringDetails);
+    return createHistoryEntry("EXPIRED", status, entity.toString());
   }
 
-  protected BeekeeperHistory createUnreferencedEventDetails(HousekeepingStatus status) throws JsonProcessingException {
+  protected BeekeeperHistory createUnreferencedEvent(HousekeepingStatus status) {
     HousekeepingPath entity = HousekeepingPath.builder()
         .cleanupAttempts(3)
         .cleanupDelay(PeriodDuration.parse("P1D"))
         .creationTimestamp(COLUMN_TIMESTAMP)
         .modifiedTimestamp(COLUMN_TIMESTAMP)
         .build();
 
-    String stringDetails = mapper.writeValueAsString(entity);
-    return createHistoryEntry("UNREFERENCED", status, stringDetails);
+    return createHistoryEntry("UNREFERENCED", status, entity.toString());
   }
 
   protected BeekeeperHistory createHistoryEntry(String lifecycleType, HousekeepingStatus status,

diff --git a/...re/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java b/...re/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java
@@ -17,9 +17,6 @@
 import org.mockito.junit.jupiter.MockitoExtension;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 
 import com.expediagroup.beekeeper.core.model.HousekeepingEntity;
 import com.expediagroup.beekeeper.core.model.HousekeepingMetadata;
@@ -106,11 +103,7 @@ private HousekeepingPath createHousekeepingPath() {
         .build();
   }
 
-  private String createEventDetails(HousekeepingEntity housekeepingEntity) throws JsonProcessingException {
-    ObjectMapper mapper = new ObjectMapper();
-    mapper.findAndRegisterModules();
-    mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
-    mapper.registerModule(new JavaTimeModule());
-    return mapper.writeValueAsString(housekeepingEntity);
+  private String createEventDetails(HousekeepingEntity housekeepingEntity) {
+    return housekeepingEntity.toString();
   }
 }
diff --git a/...iagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/...iagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java
@@ -215,14 +215,18 @@ public void expiredMetadataMultipleAlterPartitionTableEvents() throws SQLExcepti
   }
 
   @Test
-  public void addEventToHistoryTable() throws SQLException, IOException, URISyntaxException {
+  public void testEventAddedToHistoryTable() throws SQLException, IOException, URISyntaxException {
     CreateTableSqsMessage createTableSqsMessage = new CreateTableSqsMessage(LOCATION_A, true);
     amazonSQS.sendMessage(sendMessageRequest(createTableSqsMessage.getFormattedString()));
 
     await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(EXPIRED) == 1);
 
     List<BeekeeperHistory> beekeeperHistory = getBeekeeperHistory(EXPIRED);
-    System.out.println(beekeeperHistory);
+    BeekeeperHistory history = beekeeperHistory.get(0);
+    assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE);
+    assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE);
+    assertThat(history.getLifecycleType()).isEqualTo(EXPIRED.toString());
+    assertThat(history.getHousekeepingStatus()).isEqualTo(SCHEDULED.name());
   }
 
   @Test

diff --git a/...ts/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java b/...ts/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java
@@ -64,7 +64,6 @@
 
 import com.expediagroup.beekeeper.core.model.HousekeepingMetadata;
 import com.expediagroup.beekeeper.core.model.HousekeepingPath;
-import com.expediagroup.beekeeper.core.model.HousekeepingStatus;
 import com.expediagroup.beekeeper.core.model.LifecycleEventType;
 import com.expediagroup.beekeeper.core.model.PeriodDuration;
 import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;
@@ -205,16 +204,6 @@ protected void insertExpiredMetadata(HousekeepingMetadata metadata) throws SQLEx
             values);
   }
 
-  protected void insertBeekeeperHistory(BeekeeperHistory beekeeperHistory) throws SQLException {
-    String values = Stream.of(beekeeperHistory.getId().toString(), beekeeperHistory.getEventTimestamp(),
-            beekeeperHistory.getDatabaseName(), beekeeperHistory.getTableName(), beekeeperHistory.getLifecycleType(),
-            beekeeperHistory.getHousekeepingStatus(), beekeeperHistory.getEventDetails())
-        .map(s -> s == null ? null : "\"" + s + "\"")
-        .collect(Collectors.joining(", "));
-
-    mySQLTestUtils.insertToTable(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, BEEKEEPER_HISTORY_FIELDS, values);
-  }
-
   protected int getUnreferencedPathsRowCount() throws SQLException {
     return mySQLTestUtils
         .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME,
@@ -234,7 +223,7 @@ protected int getUpdatedExpiredMetadataRowCount() throws SQLException {
   }
 
   protected int getBeekeeperHistoryRowCount(LifecycleEventType lifecycleEventType) throws SQLException {
-    String filter  = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'";
+    String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'";
 
     return mySQLTestUtils.getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME,
         format(filter, lifecycleEventType));
@@ -267,7 +256,7 @@ protected List<HousekeepingMetadata> getExpiredMetadata() throws SQLException {
   }
 
   protected List<BeekeeperHistory> getBeekeeperHistory(LifecycleEventType lifecycleEventType) throws SQLException {
-    String filter  = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'";
+    String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'";
     List<BeekeeperHistory> history = new ArrayList<>();
     ResultSet resultSet = mySQLTestUtils.getTableRows(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME,
         format(filter, lifecycleEventType));
@@ -318,21 +307,4 @@ private HousekeepingMetadata createHousekeepingMetadata(
         .clientId(CLIENT_ID_FIELD)
         .build();
   }
-
-  private BeekeeperHistory createBeekeeperHistory(
-      String tableName,
-      LifecycleEventType lifecycleEventType,
-      HousekeepingStatus housekeepingStatus,
-      String eventDetails
-  ) {
-    return BeekeeperHistory.builder()
-        .id(id++)
-        .eventTimestamp(CREATION_TIMESTAMP_VALUE)
-        .databaseName(DATABASE_NAME_VALUE)
-        .tableName(tableName)
-        .lifecycleType(lifecycleEventType.toString())
-        .housekeepingStatus(housekeepingStatus.name())
-        .eventDetails(eventDetails)
-        .build();
-  }
 }
diff --git a/.../java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java b/.../java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java
@@ -26,6 +26,9 @@
 import static com.expediagroup.beekeeper.cleanup.monitoring.DeletedMetadataReporter.METRIC_NAME;
 import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED;
 import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DISABLED;
+import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED;
+import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED;
+import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.LONG_CLEANUP_DELAY_VALUE;
@@ -67,6 +70,8 @@
 import com.google.common.collect.ImmutableMap;
 
 import com.expediagroup.beekeeper.cleanup.monitoring.BytesDeletedReporter;
+import com.expediagroup.beekeeper.core.model.LifecycleEventType;
+import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;
 import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils;
 import com.expediagroup.beekeeper.integration.utils.HiveTestUtils;
 import com.expediagroup.beekeeper.metadata.cleanup.BeekeeperMetadataCleanup;
@@ -368,6 +373,27 @@ public void onlyCleanupLocationWhenPartitionExists() throws TException, SQLExcep
     assertThat(amazonS3.doesObjectExist(BUCKET, PARTITIONED_OBJECT_KEY)).isTrue();
   }
 
+  @Test
+  public void testEventAddedToHistoryTable() throws TException, SQLException {
+    hiveTestUtils.createTable(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false);
+    amazonS3.putObject(BUCKET, UNPARTITIONED_OBJECT_KEY, TABLE_DATA);
+
+    insertExpiredMetadata(UNPARTITIONED_TABLE_PATH, null);
+    await()
+        .atMost(TIMEOUT, TimeUnit.SECONDS)
+        .until(() -> getBeekeeperHistoryRowCount(LifecycleEventType.EXPIRED) == 1);
+
+    assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isFalse();
+    assertThat(amazonS3.doesObjectExist(BUCKET, UNPARTITIONED_OBJECT_KEY)).isFalse();
+
+    List<BeekeeperHistory> beekeeperHistory = getBeekeeperHistory(UNREFERENCED);
+    BeekeeperHistory history = beekeeperHistory.get(0);
+    assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE);
+    assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE);
+    assertThat(history.getLifecycleType()).isEqualTo(EXPIRED.toString());
+    assertThat(history.getHousekeepingStatus()).isEqualTo(DELETED.name());
+  }
+
   @Test
   public void metrics() throws Exception {
     Table table = hiveTestUtils.createTable(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true);

diff --git a/...test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java b/...test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java
@@ -21,6 +21,9 @@
 
 import static com.expediagroup.beekeeper.cleanup.monitoring.BytesDeletedReporter.METRIC_NAME;
 import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED;
+import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED;
+import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED;
+import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE;
@@ -52,6 +55,7 @@
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.CreateBucketRequest;
 
+import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;
 import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils;
 import com.expediagroup.beekeeper.path.cleanup.BeekeeperPathCleanup;
 
@@ -244,6 +248,30 @@ public void cleanupSentinelForNonEmptyParent() throws SQLException {
     assertThat(amazonS3.doesObjectExist(BUCKET, tableSentinel)).isTrue();
   }
 
+  @Test
+  public void testEventAddedToHistoryTable() throws SQLException {
+    amazonS3.putObject(BUCKET, OBJECT_KEY1, CONTENT);
+    amazonS3.putObject(BUCKET, OBJECT_KEY_OTHER, CONTENT);
+    amazonS3.putObject(BUCKET, OBJECT_KEY_SENTINEL, "");
+
+    String path = "s3://" + BUCKET + "/" + OBJECT_KEY1;
+    insertUnreferencedPath(path);
+
+    await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1);
+
+    assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse();
+    // deleting a file shouldn't delete a folder sentinel
+    assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY_SENTINEL)).isTrue();
+    assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY_OTHER)).isTrue();
+
+    List<BeekeeperHistory> beekeeperHistory = getBeekeeperHistory(UNREFERENCED);
+    BeekeeperHistory history = beekeeperHistory.get(0);
+    assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE);
+    assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE);
+    assertThat(history.getLifecycleType()).isEqualTo(EXPIRED.toString());
+    assertThat(history.getHousekeepingStatus()).isEqualTo(DELETED.name());
+  }
+
   @Test
   public void metrics() throws SQLException {
     amazonS3.putObject(BUCKET, OBJECT_KEY1, CONTENT);

diff --git a/...agroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/...agroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java
@@ -199,15 +199,19 @@ public void unreferencedDropTableEvent() throws SQLException, IOException, URISy
   }
 
   @Test
-  public void addBeekeeperHistoryEvent() throws IOException, URISyntaxException, SQLException {
+  public void testEventAddedToHistoryTable() throws IOException, URISyntaxException, SQLException {
     AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation",
         "s3://bucket/oldTableLocation", true, true);
     amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString()));
 
     await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1);
 
-    List<BeekeeperHistory> beekeeperHistory = getBeekeeperHistory(EXPIRED);
-    System.out.println(beekeeperHistory);
+    List<BeekeeperHistory> beekeeperHistory = getBeekeeperHistory(UNREFERENCED);
+    BeekeeperHistory history = beekeeperHistory.get(0);
+    assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE);
+    assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE);
+    assertThat(history.getLifecycleType()).isEqualTo(EXPIRED.toString());
+    assertThat(history.getHousekeepingStatus()).isEqualTo(SCHEDULED.name());
   }
 
   @Test