From 77b421c636bf79bcef3a277af10ee8fed7c918dd Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Thu, 2 Nov 2023 15:04:05 +0000 Subject: [PATCH 01/33] fix:API-paging Updating converter classes and adding tests for paging --- .../response/MetadataResponseConverter.java | 11 ++- .../api/response/PathResponseConverter.java | 6 +- .../HousekeepingMetadataResponseTest.java | 4 +- .../response/PathResponseConverterTest.java | 53 ++++++++++ .../DummyHousekeepingEntityGenerator.java | 16 +++ .../api/BeekeeperApiIntegrationTest.java | 98 ++++++++++++++++++- 6 files changed, 182 insertions(+), 6 deletions(-) create mode 100644 beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/PathResponseConverterTest.java diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/MetadataResponseConverter.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/MetadataResponseConverter.java index 097e5ef6..5aa4e2fb 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/MetadataResponseConverter.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/MetadataResponseConverter.java @@ -37,13 +37,18 @@ public final class MetadataResponseConverter { public static Page convertToHousekeepingMetadataResponsePage( Page housekeepingMetadataPage) { List housekeepingMetadataList = housekeepingMetadataPage.getContent(); + //Create a new ArrayList to store the HousekeepingMetadataResponse objects. List housekeepingMetadataResponseList = new ArrayList<>(); + //Iterate over the list of HousekeepingMetadata objects and convert each one to a HousekeepingMetadataResponse object. for (HousekeepingMetadata housekeepingMetadata : housekeepingMetadataList) { - HousekeepingMetadataResponse housekeepingMetadataResponse = convertToHousekeepingMetadataResponse( - housekeepingMetadata); + HousekeepingMetadataResponse housekeepingMetadataResponse = convertToHousekeepingMetadataResponse(housekeepingMetadata); housekeepingMetadataResponseList.add(housekeepingMetadataResponse); } - return new PageImpl<>(housekeepingMetadataResponseList); + //Create a new PageImpl object using the HousekeepingMetadataResponse list, the Pageable + // object from the HousekeepingMetadataPage object, and the total number of elements from the HousekeepingMetadataPage object. + PageImpl housekeepingMetadataResponses = new PageImpl<>( + housekeepingMetadataResponseList, housekeepingMetadataPage.getPageable(), housekeepingMetadataPage.getTotalElements()); + return housekeepingMetadataResponses; } private static HousekeepingMetadataResponse convertToHousekeepingMetadataResponse( diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java index aa82949a..45ab128b 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java @@ -51,12 +51,16 @@ private static HousekeepingPathResponse convertToHousekeepingPathResponse(Housek public static Page convertToHousekeepingPathResponsePage( Page housekeepingPathPage) { List housekeepingPathList = housekeepingPathPage.getContent(); + //Create a new ArrayList to store the HousekeepingPathResponse objects. List housekeepingPathResponseList = new ArrayList<>(); + //Iterate over the list of HousekeepingPath objects and convert each one to a HousekeepingPathResponse object. for (HousekeepingPath housekeepingPath : housekeepingPathList) { HousekeepingPathResponse housekeepingPathResponse = convertToHousekeepingPathResponse(housekeepingPath); housekeepingPathResponseList.add(housekeepingPathResponse); } - return new PageImpl<>(housekeepingPathResponseList); + PageImpl HousekeepingPathResponses = new PageImpl<>( + housekeepingPathResponseList, housekeepingPathPage.getPageable(), housekeepingPathPage.getTotalElements()); + return HousekeepingPathResponses; } } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponseTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponseTest.java index 0835404a..c35c3b90 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponseTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponseTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2021 Expedia, Inc. + * Copyright (C) 2019-2023 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,6 +46,8 @@ public void testConvertToHouseKeepingMetadataResponsePage() { assertThat(metadataResponsePageList.get(0)).isEqualTo(metadataResponse1); assertThat(metadataResponsePageList.get(1)).isEqualTo(metadataResponse2); + assertThat(metadataResponsePage.getTotalElements()).isEqualTo(2L); + assertThat(metadataResponsePage.getTotalPages()).isEqualTo(1L); assertThat(metadataResponsePage.getPageable()).isEqualTo((new PageImpl<>(housekeepingMetadataList).getPageable())); } diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/PathResponseConverterTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/PathResponseConverterTest.java new file mode 100644 index 00000000..ecc34924 --- /dev/null +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/PathResponseConverterTest.java @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2019-2023 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.api.response; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +import static com.expediagroup.beekeeper.api.response.PathResponseConverter.convertToHousekeepingPathResponsePage; +import static com.expediagroup.beekeeper.api.util.DummyHousekeepingEntityGenerator.generateDummyHousekeepingPath; +import static com.expediagroup.beekeeper.api.util.DummyHousekeepingEntityGenerator.generateDummyHousekeepingPathResponse; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; + +import com.expediagroup.beekeeper.core.model.HousekeepingPath; + +public class PathResponseConverterTest { + + @Test + public void testConvertToHousekeepingPathResponsePage() { + HousekeepingPath housekeepingPath1 = generateDummyHousekeepingPath("some_database1", "some_table1"); + HousekeepingPath housekeepingPath2 = generateDummyHousekeepingPath("some_database2", "some_table2"); + HousekeepingPathResponse housekeepingPathResponse1 = generateDummyHousekeepingPathResponse(housekeepingPath1); + HousekeepingPathResponse housekeepingPathResponse2 = generateDummyHousekeepingPathResponse(housekeepingPath2); + + List housekeepingPathList = List.of(housekeepingPath1, housekeepingPath2); + Page housekeepingPathResponsePage = convertToHousekeepingPathResponsePage( + new PageImpl<>(housekeepingPathList)); + + List housekeepingPathResponsePageList = housekeepingPathResponsePage.getContent(); + + assertThat(housekeepingPathResponsePageList.get(0)).isEqualTo(housekeepingPathResponse1); + assertThat(housekeepingPathResponsePageList.get(1)).isEqualTo(housekeepingPathResponse2); + assertThat(housekeepingPathResponsePage.getTotalElements()).isEqualTo(2L); + assertThat(housekeepingPathResponsePage.getTotalPages()).isEqualTo(1L); + assertThat(housekeepingPathResponsePage.getPageable()).isEqualTo((new PageImpl<>(housekeepingPathList).getPageable())); + } +} \ No newline at end of file diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingEntityGenerator.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingEntityGenerator.java index bd1da5ce..9d8ac397 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingEntityGenerator.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/util/DummyHousekeepingEntityGenerator.java @@ -23,6 +23,7 @@ import java.time.ZoneId; import com.expediagroup.beekeeper.api.response.HousekeepingMetadataResponse; +import com.expediagroup.beekeeper.api.response.HousekeepingPathResponse; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.PeriodDuration; @@ -88,4 +89,19 @@ public static HousekeepingPath generateDummyHousekeepingPath(String databaseName .build(); } + public static HousekeepingPathResponse generateDummyHousekeepingPathResponse(HousekeepingPath housekeepingPath) { + return HousekeepingPathResponse + .builder() + .path("s3://some/path/") + .databaseName(housekeepingPath.getDatabaseName()) + .tableName(housekeepingPath.getTableName()) + .housekeepingStatus(SCHEDULED) + .creationTimestamp(CREATION_TIMESTAMP) + .modifiedTimestamp(CREATION_TIMESTAMP) + .cleanupTimestamp(housekeepingPath.getCleanupTimestamp()) + .cleanupDelay(CLEANUP_DELAY_STRING) + .cleanupAttempts(0) + .lifecycleType(EXPIRED.toString()) + .build(); + } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index a82e8c55..2110a3cc 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -181,11 +181,107 @@ public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedE Page responsePage = mapper .readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); - + assertThat(responsePage.getTotalElements()).isEqualTo(1L); assertThatPathsEqualsResponse(testPath1, result.get(0)); assertThat(result.size()).isEqualTo(1); } + @Test + public void testPathsPageable() throws SQLException, InterruptedException, IOException { + // Test the total elements on different pages by setting the page size to 1. + int pageSize = 1; + + // Create three new HousekeepingPath objects + HousekeepingPath testPath1 = createHousekeepingPath("some_table", + "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A", LifecycleEventType.EXPIRED, + Duration.parse("P3D").toString()); + HousekeepingPath testPath2 = createHousekeepingPath("some_table", + "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B", LifecycleEventType.UNREFERENCED, + Duration.parse("P3D").toString()); + HousekeepingPath testPath3 = createHousekeepingPath("some_table", + "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=D", LifecycleEventType.UNREFERENCED, + Duration.parse("P3D").toString()); + + // Set the housekeepingStatus and cleanupTimestamp properties + testPath1.setHousekeepingStatus(HousekeepingStatus.FAILED); + testPath2.setHousekeepingStatus(HousekeepingStatus.FAILED); + testPath3.setHousekeepingStatus(HousekeepingStatus.FAILED); + testPath1.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); + testPath2.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); + testPath3.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); + + // Insert the three objects into the database + insertUnreferencedPath(testPath1); + insertUnreferencedPath(testPath2); + insertUnreferencedPath(testPath3); + + // Call the getUnreferencedPaths() method with the relevant filters + String filters = "?housekeeping_status=FAILED&page=1&size=" + pageSize; // Concat the pageSize set initially + HttpResponse response = testClient.getUnreferencedPaths("some_database", "some_table", filters); + + // Assert that the response contains the entries in the database + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + List result = responsePage.getContent(); + + // Assert that each page contains exactly one element + assertThat(result.size()).isEqualTo(1); + // Assert that there are a total of three elements + assertThat(responsePage.getTotalElements()).isEqualTo(3L); + // Assert that there are a total of three pages. + assertThat(responsePage.getTotalPages()).isEqualTo(3L); + } + + @Test + public void testMetadataPageable() throws SQLException, InterruptedException, IOException { + // Test the total elements on different pages by setting the page size to 1. + int pageSize = 1; + + // Create three new HousekeepingMetadata objects + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("some_table", + "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A", + "event_date=2020-01-01/event_hour=0/event_type=A", LifecycleEventType.EXPIRED, "P3D"); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("some_table", + "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B", + "event_date=2020-01-01/event_hour=0/event_type=B", LifecycleEventType.UNREFERENCED, "P3D"); + HousekeepingMetadata testMetadata3 = createHousekeepingMetadata("some_table", + "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=C", + "event_date=2020-01-01/event_hour=0/event_type=C", LifecycleEventType.UNREFERENCED, "P3D"); + + // Set the housekeepingStatus and cleanupTimestamp properties + testMetadata1.setHousekeepingStatus(HousekeepingStatus.FAILED); + testMetadata2.setHousekeepingStatus(HousekeepingStatus.FAILED); + testMetadata3.setHousekeepingStatus(HousekeepingStatus.FAILED); + testMetadata1.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); + testMetadata2.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); + testMetadata3.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); + + // Insert the three objects into the database + insertExpiredMetadata(testMetadata1); + insertExpiredMetadata(testMetadata2); + insertExpiredMetadata(testMetadata3); + + // Call the getUnreferencedPaths() method with the relevant filters + String filters = "?housekeeping_status=FAILED&page=1&size=" + pageSize; // Concat the pageSize set initially + HttpResponse response = testClient.getMetadata("some_database", "some_table", filters); + + // Assert that the response contains the entries in the database + assertThat(response.statusCode()).isEqualTo(OK.value()); + String body = response.body(); + Page responsePage = mapper + .readValue(body, new TypeReference>() {}); + List result = responsePage.getContent(); + + // Assert that each page contains exactly one element + assertThat(result.size()).isEqualTo(1); + // Assert that there are a total of three elements + assertThat(responsePage.getTotalElements()).isEqualTo(3L); + // Assert that there are a total of three pages. + assertThat(responsePage.getTotalPages()).isEqualTo(3L); + } + // This test is to manually test the API @Disabled @Test From f5e476d0944535c85cd2dd49fb4bfbda4747ecde Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Fri, 3 Nov 2023 12:49:10 +0000 Subject: [PATCH 02/33] Adding tests Added test to HousekeepingMetadataResponseTest to ensure paging works with multiple pages and elements add another test where these values are not the default ones (more than 1 page and total num of elements is higher than the page size) --- .../HousekeepingMetadataResponseTest.java | 20 ++ .../api/BeekeeperApiIntegrationTest.java | 185 +++++++----------- 2 files changed, 92 insertions(+), 113 deletions(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponseTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponseTest.java index c35c3b90..663bb7a4 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponseTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponseTest.java @@ -21,11 +21,13 @@ import static com.expediagroup.beekeeper.api.util.DummyHousekeepingEntityGenerator.generateDummyHousekeepingMetadata; import static com.expediagroup.beekeeper.api.util.DummyHousekeepingEntityGenerator.generateDummyHousekeepingMetadataResponse; +import java.util.ArrayList; import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.PageRequest; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; @@ -51,4 +53,22 @@ public void testConvertToHouseKeepingMetadataResponsePage() { assertThat(metadataResponsePage.getPageable()).isEqualTo((new PageImpl<>(housekeepingMetadataList).getPageable())); } + @Test + public void testConvertToHouseKeepingMetadataResponsePageWithMultiplePages() { + // Create a list of housekeeping metadata objects that is larger than the page size + List housekeepingMetadataList = new ArrayList<>(); + for (int i = 0; i < 50; i++) { + housekeepingMetadataList.add(generateDummyHousekeepingMetadata("some_database" + i, "some_table" + i)); + } + + // Create a page of housekeeping metadata objects + Page metadataPage = new PageImpl<>(housekeepingMetadataList, PageRequest.of(0, 10), 50); + + // Convert the page of housekeeping metadata objects to a page of housekeeping metadata response objects + Page metadataResponsePage = convertToHousekeepingMetadataResponsePage(metadataPage); + + // Assert that the housekeeping metadata response page has the correct total elements and total pages + assertThat(metadataResponsePage.getTotalElements()).isEqualTo(50L); + assertThat(metadataResponsePage.getTotalPages()).isEqualTo(5L); + } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index 2110a3cc..b80b5801 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -31,6 +31,7 @@ import java.sql.SQLException; import java.time.Duration; import java.time.LocalDateTime; +import java.util.Arrays; import java.util.List; import org.junit.jupiter.api.AfterEach; @@ -75,6 +76,16 @@ public ObjectMapper geObjectMapper() { protected final ObjectMapper mapper = geObjectMapper(); private Long id = 1L; + protected static final String someTable = "some_table"; + protected static final String someDatabase = "some_database"; + protected static final String pathA = "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A"; + protected static final String pathB = "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B"; + protected static final String pathC = "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=C"; + protected static final Duration duration = Duration.parse("P3D"); + protected static final int pageSize = 1; + protected static final String partitionA = "event_date=2020-01-01/event_hour=0/event_type=A"; + protected static final String partitionB = "event_date=2020-01-01/event_hour=0/event_type=B"; + protected static final String partitionC = "event_date=2020-01-01/event_hour=0/event_type=C"; @BeforeEach public void beforeEach() { @@ -98,12 +109,8 @@ public final void afterEach() { @Test public void testGetMetadataWhenTableNotFoundReturnsEmptyList() throws SQLException, InterruptedException, IOException { - HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A", - "event_date=2020-01-01/event_hour=0/event_type=A", LifecycleEventType.EXPIRED, "P3D"); - HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B", - "event_date=2020-01-01/event_hour=0/event_type=B", LifecycleEventType.EXPIRED, "P3D"); + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, duration.toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.EXPIRED, duration.toString()); insertExpiredMetadata(testMetadata1); insertExpiredMetadata(testMetadata2); @@ -117,21 +124,17 @@ public void testGetMetadataWhenTableNotFoundReturnsEmptyList() @Test public void testGetMetadataWhenThereIsFiltering() throws SQLException, InterruptedException, IOException { - HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A", - "event_date=2020-01-01/event_hour=0/event_type=A", LifecycleEventType.EXPIRED, "P3D"); - HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B", - "event_date=2020-01-01/event_hour=0/event_type=B", LifecycleEventType.UNREFERENCED, "P3D"); - HousekeepingMetadata testMetadata3 = createHousekeepingMetadata("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=C", - "event_date=2020-01-01/event_hour=0/event_type=C", LifecycleEventType.UNREFERENCED, "P3D"); + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, duration.toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.UNREFERENCED, duration.toString()); + HousekeepingMetadata testMetadata3 = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.UNREFERENCED, duration.toString()); + testMetadata1.setHousekeepingStatus(HousekeepingStatus.FAILED); testMetadata1.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); testMetadata1.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - insertExpiredMetadata(testMetadata1); - insertExpiredMetadata(testMetadata2); - insertExpiredMetadata(testMetadata3); + + for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { + insertExpiredMetadata(testMetadata); + } String filters = "?housekeeping_status=FAILED" + "&lifecycle_type=EXPIRED" @@ -140,7 +143,7 @@ public void testGetMetadataWhenThereIsFiltering() throws SQLException, Interrupt + "&path=s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A" + "&partition_name=event_date=2020-01-01/event_hour=0/event_type=A"; - HttpResponse response = testClient.getMetadata("some_database", "some_table", filters); + HttpResponse response = testClient.getMetadata(someDatabase, someTable, filters); assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); Page responsePage = mapper @@ -148,34 +151,28 @@ public void testGetMetadataWhenThereIsFiltering() throws SQLException, Interrupt List result = responsePage.getContent(); assertThatMetadataEqualsResponse(testMetadata1, result.get(0)); - assertThat(result.size()).isEqualTo(1); + assertThat(result).hasSize(1); } @Test public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedException, IOException { - HousekeepingPath testPath1 = createHousekeepingPath("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A", LifecycleEventType.EXPIRED, - Duration.parse("P3D").toString()); - HousekeepingPath testPath2 = createHousekeepingPath("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B", LifecycleEventType.UNREFERENCED, - Duration.parse("P3D").toString()); - HousekeepingPath testPath3 = createHousekeepingPath("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=C", LifecycleEventType.UNREFERENCED, - Duration.parse("P3D").toString()); + HousekeepingPath testPath1 = createHousekeepingPath(someTable, pathA, LifecycleEventType.EXPIRED, duration.toString()); + HousekeepingPath testPath2 = createHousekeepingPath(someTable, pathB, LifecycleEventType.UNREFERENCED, duration.toString()); + HousekeepingPath testPath3 = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, duration.toString()); testPath1.setHousekeepingStatus(HousekeepingStatus.FAILED); testPath1.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); testPath1.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - insertUnreferencedPath(testPath1); - insertUnreferencedPath(testPath2); - insertUnreferencedPath(testPath3); + for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { + insertUnreferencedPath(testPath); + } String filters = "?housekeeping_status=FAILED" + "&lifecycle_type=EXPIRED" + "&deleted_before=2000-05-05T10:41:20" + "®istered_before=2000-04-04T10:41:20" + "&path=s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A"; - HttpResponse response = testClient.getUnreferencedPaths("some_database", "some_table", filters); + HttpResponse response = testClient.getUnreferencedPaths(someDatabase, someTable, filters); assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); Page responsePage = mapper @@ -183,41 +180,30 @@ public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedE List result = responsePage.getContent(); assertThat(responsePage.getTotalElements()).isEqualTo(1L); assertThatPathsEqualsResponse(testPath1, result.get(0)); - assertThat(result.size()).isEqualTo(1); + assertThat(result).hasSize(1); } @Test public void testPathsPageable() throws SQLException, InterruptedException, IOException { - // Test the total elements on different pages by setting the page size to 1. - int pageSize = 1; // Create three new HousekeepingPath objects - HousekeepingPath testPath1 = createHousekeepingPath("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A", LifecycleEventType.EXPIRED, - Duration.parse("P3D").toString()); - HousekeepingPath testPath2 = createHousekeepingPath("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B", LifecycleEventType.UNREFERENCED, - Duration.parse("P3D").toString()); - HousekeepingPath testPath3 = createHousekeepingPath("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=D", LifecycleEventType.UNREFERENCED, - Duration.parse("P3D").toString()); + HousekeepingPath testPath1 = createHousekeepingPath(someTable, pathA, LifecycleEventType.EXPIRED, duration.toString()); + HousekeepingPath testPath2 = createHousekeepingPath(someTable, pathB, LifecycleEventType.UNREFERENCED, duration.toString()); + HousekeepingPath testPath3 = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, duration.toString()); - // Set the housekeepingStatus and cleanupTimestamp properties - testPath1.setHousekeepingStatus(HousekeepingStatus.FAILED); - testPath2.setHousekeepingStatus(HousekeepingStatus.FAILED); - testPath3.setHousekeepingStatus(HousekeepingStatus.FAILED); - testPath1.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - testPath2.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - testPath3.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); + // Set the housekeepingStatus for all test paths as FAILED + for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { + testPath.setHousekeepingStatus(HousekeepingStatus.FAILED); + } // Insert the three objects into the database - insertUnreferencedPath(testPath1); - insertUnreferencedPath(testPath2); - insertUnreferencedPath(testPath3); + for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { + insertUnreferencedPath(testPath); + } // Call the getUnreferencedPaths() method with the relevant filters - String filters = "?housekeeping_status=FAILED&page=1&size=" + pageSize; // Concat the pageSize set initially - HttpResponse response = testClient.getUnreferencedPaths("some_database", "some_table", filters); + String filters = "?housekeeping_status=FAILED&page=1&size=" + pageSize; + HttpResponse response = testClient.getUnreferencedPaths(someDatabase, someTable, filters); // Assert that the response contains the entries in the database assertThat(response.statusCode()).isEqualTo(OK.value()); @@ -226,46 +212,31 @@ public void testPathsPageable() throws SQLException, InterruptedException, IOExc .readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); - // Assert that each page contains exactly one element - assertThat(result.size()).isEqualTo(1); - // Assert that there are a total of three elements + assertThat(result).hasSize(1); assertThat(responsePage.getTotalElements()).isEqualTo(3L); - // Assert that there are a total of three pages. assertThat(responsePage.getTotalPages()).isEqualTo(3L); } @Test public void testMetadataPageable() throws SQLException, InterruptedException, IOException { - // Test the total elements on different pages by setting the page size to 1. - int pageSize = 1; - // Create three new HousekeepingMetadata objects - HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A", - "event_date=2020-01-01/event_hour=0/event_type=A", LifecycleEventType.EXPIRED, "P3D"); - HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B", - "event_date=2020-01-01/event_hour=0/event_type=B", LifecycleEventType.UNREFERENCED, "P3D"); - HousekeepingMetadata testMetadata3 = createHousekeepingMetadata("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=C", - "event_date=2020-01-01/event_hour=0/event_type=C", LifecycleEventType.UNREFERENCED, "P3D"); + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, duration.toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.EXPIRED, duration.toString()); + HousekeepingMetadata testMetadata3 = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.EXPIRED, duration.toString()); // Set the housekeepingStatus and cleanupTimestamp properties testMetadata1.setHousekeepingStatus(HousekeepingStatus.FAILED); testMetadata2.setHousekeepingStatus(HousekeepingStatus.FAILED); testMetadata3.setHousekeepingStatus(HousekeepingStatus.FAILED); - testMetadata1.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - testMetadata2.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - testMetadata3.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); // Insert the three objects into the database - insertExpiredMetadata(testMetadata1); - insertExpiredMetadata(testMetadata2); - insertExpiredMetadata(testMetadata3); + for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { + insertExpiredMetadata(testMetadata); + } // Call the getUnreferencedPaths() method with the relevant filters - String filters = "?housekeeping_status=FAILED&page=1&size=" + pageSize; // Concat the pageSize set initially - HttpResponse response = testClient.getMetadata("some_database", "some_table", filters); + String filters = "?housekeeping_status=FAILED&page=1&size=" + pageSize; + HttpResponse response = testClient.getMetadata(someDatabase, someTable, filters); // Assert that the response contains the entries in the database assertThat(response.statusCode()).isEqualTo(OK.value()); @@ -274,11 +245,8 @@ public void testMetadataPageable() throws SQLException, InterruptedException, IO .readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); - // Assert that each page contains exactly one element - assertThat(result.size()).isEqualTo(1); - // Assert that there are a total of three elements + assertThat(result).hasSize(1); assertThat(responsePage.getTotalElements()).isEqualTo(3L); - // Assert that there are a total of three pages. assertThat(responsePage.getTotalPages()).isEqualTo(3L); } @@ -286,34 +254,25 @@ public void testMetadataPageable() throws SQLException, InterruptedException, IO @Disabled @Test public void manualTest() throws SQLException, InterruptedException { - HousekeepingMetadata testMetadata1 = createHousekeepingMetadata("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A", - "event_date=2020-01-01/event_hour=0/event_type=A", LifecycleEventType.EXPIRED, - Duration.parse("P3D").toString()); - HousekeepingMetadata testMetadata2 = createHousekeepingMetadata("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B", - "event_date=2020-01-01/event_hour=0/event_type=B", LifecycleEventType.EXPIRED, - Duration.parse("P3D").toString()); - HousekeepingMetadata testMetadata3 = createHousekeepingMetadata("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=C", - "event_date=2020-01-01/event_hour=0/event_type=C", LifecycleEventType.UNREFERENCED, - Duration.parse("P4D").toString()); - insertExpiredMetadata(testMetadata1); - insertExpiredMetadata(testMetadata2); - insertExpiredMetadata(testMetadata3); - - HousekeepingPath testPath1 = createHousekeepingPath("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=A", LifecycleEventType.EXPIRED, - Duration.parse("P3D").toString()); - HousekeepingPath testPath2 = createHousekeepingPath("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=B", LifecycleEventType.UNREFERENCED, - Duration.parse("P3D").toString()); - HousekeepingPath testPath3 = createHousekeepingPath("some_table", - "s3://some/path/event_date=2020-01-01/event_hour=0/event_type=C", LifecycleEventType.UNREFERENCED, - Duration.parse("P3D").toString()); - insertUnreferencedPath(testPath1); - insertUnreferencedPath(testPath2); - insertUnreferencedPath(testPath3); + HousekeepingMetadata testMetadata1 = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, + duration.toString()); + HousekeepingMetadata testMetadata2 = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.EXPIRED, + duration.toString()); + HousekeepingMetadata testMetadata3 = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.UNREFERENCED, + duration.toString()); + for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { + insertExpiredMetadata(testMetadata); + } + + HousekeepingPath testPath1 = createHousekeepingPath(someTable, pathA, LifecycleEventType.EXPIRED, + duration.toString()); + HousekeepingPath testPath2 = createHousekeepingPath(someTable, pathB, LifecycleEventType.UNREFERENCED, + duration.toString()); + HousekeepingPath testPath3 = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, + duration.toString()); + for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { + insertUnreferencedPath(testPath); + } Thread.sleep(10000000L); } From 89a5d58b55e045af51f20cfcd044fecfa394f296 Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Fri, 3 Nov 2023 15:16:51 +0000 Subject: [PATCH 03/33] Update BeekeeperApiIntegrationTest.java --- .../integration/api/BeekeeperApiIntegrationTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index b80b5801..1e128024 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -225,9 +225,9 @@ public void testMetadataPageable() throws SQLException, InterruptedException, IO HousekeepingMetadata testMetadata3 = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.EXPIRED, duration.toString()); // Set the housekeepingStatus and cleanupTimestamp properties - testMetadata1.setHousekeepingStatus(HousekeepingStatus.FAILED); - testMetadata2.setHousekeepingStatus(HousekeepingStatus.FAILED); - testMetadata3.setHousekeepingStatus(HousekeepingStatus.FAILED); + for (HousekeepingMetadata testPath : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { + testPath.setHousekeepingStatus(HousekeepingStatus.FAILED); + } // Insert the three objects into the database for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { From 9e7f763882e4843a4ca99f9a70f1b07358a85d15 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:32:49 +0000 Subject: [PATCH 04/33] Update CHANGELOG.md --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c77c347d..7c56886b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.5.5] - 2023-11-07 +### Changed +- Updated 'MetadataResponseConverter' and 'PathResponseConverter' to fix the conversion of the Pageable object. + +### Fixed +- Updated existing and added new unit and integration tests + ## [3.5.4] - 2023-09-14 ### Fixed - Added localisation normalization so locations like `s3:/a/b` and `s3:/a/b/` will be considered the same and path won't be scheduled for deletion. From 1a59b0237c076bd0cd694351a607dfa416b18e9b Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:34:18 +0000 Subject: [PATCH 05/33] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c56886b..a10b52aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated 'MetadataResponseConverter' and 'PathResponseConverter' to fix the conversion of the Pageable object. ### Fixed -- Updated existing and added new unit and integration tests +- Updated existing and added new unit and integration tests. ## [3.5.4] - 2023-09-14 ### Fixed From 6db54658d37574cfa919108f2419dbb377d4d3ad Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:45:48 +0000 Subject: [PATCH 06/33] Update CHANGELOG.md Co-authored-by: Andreea Paduraru <40387422+andreeapad@users.noreply.github.com> --- CHANGELOG.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a10b52aa..220f8bbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [3.5.5] - 2023-11-07 -### Changed -- Updated 'MetadataResponseConverter' and 'PathResponseConverter' to fix the conversion of the Pageable object. - ### Fixed -- Updated existing and added new unit and integration tests. +- Fixed paged API response by updating 'MetadataResponseConverter' and 'PathResponseConverter' to pass complete information about the number of pages and elements to the response Page. ## [3.5.4] - 2023-09-14 ### Fixed From 4966b560733c177b7a538bcfffac69ca1927d36b Mon Sep 17 00:00:00 2001 From: eg-oss-ci Date: Tue, 7 Nov 2023 18:10:07 +0000 Subject: [PATCH 07/33] [maven-release-plugin] prepare release beekeeper-parent-3.5.5 --- beekeeper-api/pom.xml | 2 +- beekeeper-cleanup/pom.xml | 2 +- beekeeper-core/pom.xml | 2 +- beekeeper-integration-tests/pom.xml | 2 +- beekeeper-metadata-cleanup/pom.xml | 2 +- beekeeper-path-cleanup/pom.xml | 2 +- beekeeper-scheduler-apiary/pom.xml | 2 +- beekeeper-scheduler/pom.xml | 2 +- beekeeper-vacuum-tool/pom.xml | 2 +- pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index 99d005fa..057be304 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -4,7 +4,7 @@ beekeeper-parent com.expediagroup - 3.5.5-SNAPSHOT + 3.5.5 beekeeper-api diff --git a/beekeeper-cleanup/pom.xml b/beekeeper-cleanup/pom.xml index 771cde80..40e7cbac 100644 --- a/beekeeper-cleanup/pom.xml +++ b/beekeeper-cleanup/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5-SNAPSHOT + 3.5.5 beekeeper-cleanup diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index ad6327ae..3b37d362 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5-SNAPSHOT + 3.5.5 beekeeper-core diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml index 472bcd21..f639ba6c 100644 --- a/beekeeper-integration-tests/pom.xml +++ b/beekeeper-integration-tests/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5-SNAPSHOT + 3.5.5 beekeeper-integration-tests diff --git a/beekeeper-metadata-cleanup/pom.xml b/beekeeper-metadata-cleanup/pom.xml index 558fd620..30102fa5 100644 --- a/beekeeper-metadata-cleanup/pom.xml +++ b/beekeeper-metadata-cleanup/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5-SNAPSHOT + 3.5.5 beekeeper-metadata-cleanup diff --git a/beekeeper-path-cleanup/pom.xml b/beekeeper-path-cleanup/pom.xml index 44b74567..c912705d 100644 --- a/beekeeper-path-cleanup/pom.xml +++ b/beekeeper-path-cleanup/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5-SNAPSHOT + 3.5.5 beekeeper-path-cleanup diff --git a/beekeeper-scheduler-apiary/pom.xml b/beekeeper-scheduler-apiary/pom.xml index 74d67e52..7223dac4 100644 --- a/beekeeper-scheduler-apiary/pom.xml +++ b/beekeeper-scheduler-apiary/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5-SNAPSHOT + 3.5.5 beekeeper-scheduler-apiary diff --git a/beekeeper-scheduler/pom.xml b/beekeeper-scheduler/pom.xml index fef7331a..c87b355a 100644 --- a/beekeeper-scheduler/pom.xml +++ b/beekeeper-scheduler/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5-SNAPSHOT + 3.5.5 beekeeper-scheduler diff --git a/beekeeper-vacuum-tool/pom.xml b/beekeeper-vacuum-tool/pom.xml index 2921390b..5c3afc2d 100644 --- a/beekeeper-vacuum-tool/pom.xml +++ b/beekeeper-vacuum-tool/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5-SNAPSHOT + 3.5.5 beekeeper-vacuum-tool diff --git a/pom.xml b/pom.xml index 63322518..230bab48 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ beekeeper-parent - 3.5.5-SNAPSHOT + 3.5.5 Beekeeper is a service which manages the cleanup of tables and unreferenced S3 paths. 2019 pom @@ -31,7 +31,7 @@ scm:git:https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/ExpediaGroup/beekeeper.git https://github.com/ExpediaGroup/beekeeper - HEAD + beekeeper-parent-3.5.5 From a47309e26833793f58245d3c5a9e38ca4e6966e1 Mon Sep 17 00:00:00 2001 From: eg-oss-ci Date: Tue, 7 Nov 2023 18:10:08 +0000 Subject: [PATCH 08/33] [maven-release-plugin] prepare for next development iteration --- beekeeper-api/pom.xml | 2 +- beekeeper-cleanup/pom.xml | 2 +- beekeeper-core/pom.xml | 2 +- beekeeper-integration-tests/pom.xml | 2 +- beekeeper-metadata-cleanup/pom.xml | 2 +- beekeeper-path-cleanup/pom.xml | 2 +- beekeeper-scheduler-apiary/pom.xml | 2 +- beekeeper-scheduler/pom.xml | 2 +- beekeeper-vacuum-tool/pom.xml | 2 +- pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index 057be304..cb532dde 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -4,7 +4,7 @@ beekeeper-parent com.expediagroup - 3.5.5 + 3.5.6-SNAPSHOT beekeeper-api diff --git a/beekeeper-cleanup/pom.xml b/beekeeper-cleanup/pom.xml index 40e7cbac..56ad8aed 100644 --- a/beekeeper-cleanup/pom.xml +++ b/beekeeper-cleanup/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5 + 3.5.6-SNAPSHOT beekeeper-cleanup diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index 3b37d362..501c5264 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5 + 3.5.6-SNAPSHOT beekeeper-core diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml index f639ba6c..ef18b30c 100644 --- a/beekeeper-integration-tests/pom.xml +++ b/beekeeper-integration-tests/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5 + 3.5.6-SNAPSHOT beekeeper-integration-tests diff --git a/beekeeper-metadata-cleanup/pom.xml b/beekeeper-metadata-cleanup/pom.xml index 30102fa5..f8683152 100644 --- a/beekeeper-metadata-cleanup/pom.xml +++ b/beekeeper-metadata-cleanup/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5 + 3.5.6-SNAPSHOT beekeeper-metadata-cleanup diff --git a/beekeeper-path-cleanup/pom.xml b/beekeeper-path-cleanup/pom.xml index c912705d..05011b18 100644 --- a/beekeeper-path-cleanup/pom.xml +++ b/beekeeper-path-cleanup/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5 + 3.5.6-SNAPSHOT beekeeper-path-cleanup diff --git a/beekeeper-scheduler-apiary/pom.xml b/beekeeper-scheduler-apiary/pom.xml index 7223dac4..3de4b05f 100644 --- a/beekeeper-scheduler-apiary/pom.xml +++ b/beekeeper-scheduler-apiary/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5 + 3.5.6-SNAPSHOT beekeeper-scheduler-apiary diff --git a/beekeeper-scheduler/pom.xml b/beekeeper-scheduler/pom.xml index c87b355a..89ebee81 100644 --- a/beekeeper-scheduler/pom.xml +++ b/beekeeper-scheduler/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5 + 3.5.6-SNAPSHOT beekeeper-scheduler diff --git a/beekeeper-vacuum-tool/pom.xml b/beekeeper-vacuum-tool/pom.xml index 5c3afc2d..d7e62a4a 100644 --- a/beekeeper-vacuum-tool/pom.xml +++ b/beekeeper-vacuum-tool/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.5 + 3.5.6-SNAPSHOT beekeeper-vacuum-tool diff --git a/pom.xml b/pom.xml index 230bab48..662fc5e4 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ beekeeper-parent - 3.5.5 + 3.5.6-SNAPSHOT Beekeeper is a service which manages the cleanup of tables and unreferenced S3 paths. 2019 pom @@ -31,7 +31,7 @@ scm:git:https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/ExpediaGroup/beekeeper.git https://github.com/ExpediaGroup/beekeeper - beekeeper-parent-3.5.5 + HEAD From 16bb13d46892eb6272b4e829f9d8bd337d7bfb64 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:58:10 +0000 Subject: [PATCH 09/33] Update pom.xml --- beekeeper-api/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-api/pom.xml b/beekeeper-api/pom.xml index cb532dde..99d005fa 100644 --- a/beekeeper-api/pom.xml +++ b/beekeeper-api/pom.xml @@ -4,7 +4,7 @@ beekeeper-parent com.expediagroup - 3.5.6-SNAPSHOT + 3.5.5-SNAPSHOT beekeeper-api From 643d1caef1f67c422d6a3a5c64e18ba2c69e8d2f Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:58:33 +0000 Subject: [PATCH 10/33] Update pom.xml --- beekeeper-cleanup/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-cleanup/pom.xml b/beekeeper-cleanup/pom.xml index 56ad8aed..771cde80 100644 --- a/beekeeper-cleanup/pom.xml +++ b/beekeeper-cleanup/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.6-SNAPSHOT + 3.5.5-SNAPSHOT beekeeper-cleanup From db6a691fb2b7ba9ef0045ee6686d4d535df22352 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:58:49 +0000 Subject: [PATCH 11/33] Update pom.xml --- beekeeper-integration-tests/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-integration-tests/pom.xml b/beekeeper-integration-tests/pom.xml index ef18b30c..472bcd21 100644 --- a/beekeeper-integration-tests/pom.xml +++ b/beekeeper-integration-tests/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.6-SNAPSHOT + 3.5.5-SNAPSHOT beekeeper-integration-tests From 03d3f4bc045d7dbe6b3a0ecd82f5c09355721073 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:59:06 +0000 Subject: [PATCH 12/33] Update pom.xml --- beekeeper-metadata-cleanup/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-metadata-cleanup/pom.xml b/beekeeper-metadata-cleanup/pom.xml index f8683152..558fd620 100644 --- a/beekeeper-metadata-cleanup/pom.xml +++ b/beekeeper-metadata-cleanup/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.6-SNAPSHOT + 3.5.5-SNAPSHOT beekeeper-metadata-cleanup From 2c29d084477592cc3fd70588ca00b178ad8d8ca3 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:59:21 +0000 Subject: [PATCH 13/33] Update pom.xml --- beekeeper-scheduler-apiary/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-scheduler-apiary/pom.xml b/beekeeper-scheduler-apiary/pom.xml index 3de4b05f..74d67e52 100644 --- a/beekeeper-scheduler-apiary/pom.xml +++ b/beekeeper-scheduler-apiary/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.6-SNAPSHOT + 3.5.5-SNAPSHOT beekeeper-scheduler-apiary From 586a11f0ed338751112631599bf13094318443b2 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Wed, 8 Nov 2023 12:59:34 +0000 Subject: [PATCH 14/33] Update pom.xml --- beekeeper-vacuum-tool/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-vacuum-tool/pom.xml b/beekeeper-vacuum-tool/pom.xml index d7e62a4a..2921390b 100644 --- a/beekeeper-vacuum-tool/pom.xml +++ b/beekeeper-vacuum-tool/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.6-SNAPSHOT + 3.5.5-SNAPSHOT beekeeper-vacuum-tool From 4a18f680a5544a903e96f9871497aaeeb1d7bb27 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Wed, 8 Nov 2023 13:00:54 +0000 Subject: [PATCH 15/33] Update pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 662fc5e4..63322518 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ beekeeper-parent - 3.5.6-SNAPSHOT + 3.5.5-SNAPSHOT Beekeeper is a service which manages the cleanup of tables and unreferenced S3 paths. 2019 pom From 249c99f3064a29bebde43811fb9cd1959e3b49d2 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Wed, 8 Nov 2023 13:01:08 +0000 Subject: [PATCH 16/33] Update pom.xml --- beekeeper-scheduler/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-scheduler/pom.xml b/beekeeper-scheduler/pom.xml index 89ebee81..fef7331a 100644 --- a/beekeeper-scheduler/pom.xml +++ b/beekeeper-scheduler/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.6-SNAPSHOT + 3.5.5-SNAPSHOT beekeeper-scheduler From ccead964b574de380548df9b1794abc5e3971c7b Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Wed, 8 Nov 2023 13:01:33 +0000 Subject: [PATCH 17/33] Update pom.xml --- beekeeper-path-cleanup/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-path-cleanup/pom.xml b/beekeeper-path-cleanup/pom.xml index 05011b18..44b74567 100644 --- a/beekeeper-path-cleanup/pom.xml +++ b/beekeeper-path-cleanup/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.6-SNAPSHOT + 3.5.5-SNAPSHOT beekeeper-path-cleanup From d9e016acd55296f7f5eb0c4b7f1fe8802e5a6a21 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Wed, 8 Nov 2023 13:01:54 +0000 Subject: [PATCH 18/33] Update pom.xml --- beekeeper-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index 501c5264..ad6327ae 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -5,7 +5,7 @@ beekeeper-parent com.expediagroup - 3.5.6-SNAPSHOT + 3.5.5-SNAPSHOT beekeeper-core From 30c204070402ce5a7d4fb7b41c1105c1830ad7bf Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Wed, 8 Nov 2023 15:34:12 +0000 Subject: [PATCH 19/33] Remove surplus comments --- .../api/response/MetadataResponseConverter.java | 2 -- .../api/response/PathResponseConverter.java | 2 -- .../response/HousekeepingMetadataResponseTest.java | 5 ----- .../integration/api/BeekeeperApiIntegrationTest.java | 12 ------------ 4 files changed, 21 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/MetadataResponseConverter.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/MetadataResponseConverter.java index 5aa4e2fb..58b671f2 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/MetadataResponseConverter.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/MetadataResponseConverter.java @@ -37,9 +37,7 @@ public final class MetadataResponseConverter { public static Page convertToHousekeepingMetadataResponsePage( Page housekeepingMetadataPage) { List housekeepingMetadataList = housekeepingMetadataPage.getContent(); - //Create a new ArrayList to store the HousekeepingMetadataResponse objects. List housekeepingMetadataResponseList = new ArrayList<>(); - //Iterate over the list of HousekeepingMetadata objects and convert each one to a HousekeepingMetadataResponse object. for (HousekeepingMetadata housekeepingMetadata : housekeepingMetadataList) { HousekeepingMetadataResponse housekeepingMetadataResponse = convertToHousekeepingMetadataResponse(housekeepingMetadata); housekeepingMetadataResponseList.add(housekeepingMetadataResponse); diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java index 45ab128b..d71c80d6 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java @@ -51,9 +51,7 @@ private static HousekeepingPathResponse convertToHousekeepingPathResponse(Housek public static Page convertToHousekeepingPathResponsePage( Page housekeepingPathPage) { List housekeepingPathList = housekeepingPathPage.getContent(); - //Create a new ArrayList to store the HousekeepingPathResponse objects. List housekeepingPathResponseList = new ArrayList<>(); - //Iterate over the list of HousekeepingPath objects and convert each one to a HousekeepingPathResponse object. for (HousekeepingPath housekeepingPath : housekeepingPathList) { HousekeepingPathResponse housekeepingPathResponse = convertToHousekeepingPathResponse(housekeepingPath); housekeepingPathResponseList.add(housekeepingPathResponse); diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponseTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponseTest.java index 663bb7a4..8eb0f1aa 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponseTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/HousekeepingMetadataResponseTest.java @@ -55,19 +55,14 @@ public void testConvertToHouseKeepingMetadataResponsePage() { @Test public void testConvertToHouseKeepingMetadataResponsePageWithMultiplePages() { - // Create a list of housekeeping metadata objects that is larger than the page size List housekeepingMetadataList = new ArrayList<>(); for (int i = 0; i < 50; i++) { housekeepingMetadataList.add(generateDummyHousekeepingMetadata("some_database" + i, "some_table" + i)); } - // Create a page of housekeeping metadata objects Page metadataPage = new PageImpl<>(housekeepingMetadataList, PageRequest.of(0, 10), 50); - - // Convert the page of housekeeping metadata objects to a page of housekeeping metadata response objects Page metadataResponsePage = convertToHousekeepingMetadataResponsePage(metadataPage); - // Assert that the housekeeping metadata response page has the correct total elements and total pages assertThat(metadataResponsePage.getTotalElements()).isEqualTo(50L); assertThat(metadataResponsePage.getTotalPages()).isEqualTo(5L); } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index 1e128024..cdc75cec 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -186,32 +186,26 @@ public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedE @Test public void testPathsPageable() throws SQLException, InterruptedException, IOException { - // Create three new HousekeepingPath objects HousekeepingPath testPath1 = createHousekeepingPath(someTable, pathA, LifecycleEventType.EXPIRED, duration.toString()); HousekeepingPath testPath2 = createHousekeepingPath(someTable, pathB, LifecycleEventType.UNREFERENCED, duration.toString()); HousekeepingPath testPath3 = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, duration.toString()); - // Set the housekeepingStatus for all test paths as FAILED for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { testPath.setHousekeepingStatus(HousekeepingStatus.FAILED); } - // Insert the three objects into the database for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { insertUnreferencedPath(testPath); } - // Call the getUnreferencedPaths() method with the relevant filters String filters = "?housekeeping_status=FAILED&page=1&size=" + pageSize; HttpResponse response = testClient.getUnreferencedPaths(someDatabase, someTable, filters); - // Assert that the response contains the entries in the database assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); Page responsePage = mapper .readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); - assertThat(result).hasSize(1); assertThat(responsePage.getTotalElements()).isEqualTo(3L); assertThat(responsePage.getTotalPages()).isEqualTo(3L); @@ -219,32 +213,26 @@ public void testPathsPageable() throws SQLException, InterruptedException, IOExc @Test public void testMetadataPageable() throws SQLException, InterruptedException, IOException { - // Create three new HousekeepingMetadata objects HousekeepingMetadata testMetadata1 = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, duration.toString()); HousekeepingMetadata testMetadata2 = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.EXPIRED, duration.toString()); HousekeepingMetadata testMetadata3 = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.EXPIRED, duration.toString()); - // Set the housekeepingStatus and cleanupTimestamp properties for (HousekeepingMetadata testPath : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { testPath.setHousekeepingStatus(HousekeepingStatus.FAILED); } - // Insert the three objects into the database for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { insertExpiredMetadata(testMetadata); } - // Call the getUnreferencedPaths() method with the relevant filters String filters = "?housekeeping_status=FAILED&page=1&size=" + pageSize; HttpResponse response = testClient.getMetadata(someDatabase, someTable, filters); - // Assert that the response contains the entries in the database assertThat(response.statusCode()).isEqualTo(OK.value()); String body = response.body(); Page responsePage = mapper .readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); - assertThat(result).hasSize(1); assertThat(responsePage.getTotalElements()).isEqualTo(3L); assertThat(responsePage.getTotalPages()).isEqualTo(3L); From 7a02cc5bdd97ee0930ab0906f1ac1d92e4f4684c Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Wed, 8 Nov 2023 15:55:37 +0000 Subject: [PATCH 20/33] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 220f8bbc..37403a64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [3.5.5] - 2023-11-07 +## [3.5.5] - 2023-11-TBD ### Fixed - Fixed paged API response by updating 'MetadataResponseConverter' and 'PathResponseConverter' to pass complete information about the number of pages and elements to the response Page. From bad68e04114417b9039dcaf00ae94e39a7bdf7a1 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:33:10 +0000 Subject: [PATCH 21/33] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37403a64..f361711d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [3.5.5] - 2023-11-TBD +## [3.5.5] - 2023-11-09 ### Fixed - Fixed paged API response by updating 'MetadataResponseConverter' and 'PathResponseConverter' to pass complete information about the number of pages and elements to the response Page. From 4496f4d797c2cfc828741f6984e1ccd998f275f8 Mon Sep 17 00:00:00 2001 From: Andreea Paduraru <40387422+andreeapad@users.noreply.github.com> Date: Thu, 9 Nov 2023 16:16:41 +0000 Subject: [PATCH 22/33] Update beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/PathResponseConverterTest.java --- .../beekeeper/api/response/PathResponseConverterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/PathResponseConverterTest.java b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/PathResponseConverterTest.java index ecc34924..a36e346a 100644 --- a/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/PathResponseConverterTest.java +++ b/beekeeper-api/src/test/java/com/expediagroup/beekeeper/api/response/PathResponseConverterTest.java @@ -50,4 +50,4 @@ public void testConvertToHousekeepingPathResponsePage() { assertThat(housekeepingPathResponsePage.getTotalPages()).isEqualTo(1L); assertThat(housekeepingPathResponsePage.getPageable()).isEqualTo((new PageImpl<>(housekeepingPathList).getPageable())); } -} \ No newline at end of file +} From fcc6a73886d830f6750c5c3760bd4a754b2069d0 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Thu, 9 Nov 2023 18:32:15 +0000 Subject: [PATCH 23/33] Update beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java Co-authored-by: Abhimanyu Gupta --- .../beekeeper/api/response/PathResponseConverter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java index d71c80d6..abec93ca 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java @@ -56,7 +56,7 @@ public static Page convertToHousekeepingPathResponsePa HousekeepingPathResponse housekeepingPathResponse = convertToHousekeepingPathResponse(housekeepingPath); housekeepingPathResponseList.add(housekeepingPathResponse); } - PageImpl HousekeepingPathResponses = new PageImpl<>( + PageImpl housekeepingPathResponses = new PageImpl<>( housekeepingPathResponseList, housekeepingPathPage.getPageable(), housekeepingPathPage.getTotalElements()); return HousekeepingPathResponses; } From 9d913fe6213e4f6a734df5d9224bbff3ef62cc1d Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Thu, 9 Nov 2023 18:35:52 +0000 Subject: [PATCH 24/33] Update MetadataResponseConverter.java --- .../beekeeper/api/response/MetadataResponseConverter.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/MetadataResponseConverter.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/MetadataResponseConverter.java index 58b671f2..d03c0d6c 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/MetadataResponseConverter.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/MetadataResponseConverter.java @@ -42,8 +42,6 @@ public static Page convertToHousekeepingMetadataRe HousekeepingMetadataResponse housekeepingMetadataResponse = convertToHousekeepingMetadataResponse(housekeepingMetadata); housekeepingMetadataResponseList.add(housekeepingMetadataResponse); } - //Create a new PageImpl object using the HousekeepingMetadataResponse list, the Pageable - // object from the HousekeepingMetadataPage object, and the total number of elements from the HousekeepingMetadataPage object. PageImpl housekeepingMetadataResponses = new PageImpl<>( housekeepingMetadataResponseList, housekeepingMetadataPage.getPageable(), housekeepingMetadataPage.getTotalElements()); return housekeepingMetadataResponses; From ebacb6d708da702b7f0aaa85a0ef0ebedd92d6fb Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Thu, 9 Nov 2023 22:29:32 +0000 Subject: [PATCH 25/33] Update PathResponseConverter.java --- .../beekeeper/api/response/PathResponseConverter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java index abec93ca..b948610f 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/response/PathResponseConverter.java @@ -58,7 +58,7 @@ public static Page convertToHousekeepingPathResponsePa } PageImpl housekeepingPathResponses = new PageImpl<>( housekeepingPathResponseList, housekeepingPathPage.getPageable(), housekeepingPathPage.getTotalElements()); - return HousekeepingPathResponses; + return housekeepingPathResponses; } } From 8f94433410c5e5fad264ff76b6857c9507f34464 Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Thu, 9 Nov 2023 23:00:34 +0000 Subject: [PATCH 26/33] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f361711d..826aefa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [3.5.5] - 2023-11-09 +## [3.5.5] - 2023-11-10 ### Fixed - Fixed paged API response by updating 'MetadataResponseConverter' and 'PathResponseConverter' to pass complete information about the number of pages and elements to the response Page. From e1de7604308496f75d0aef13a591d86789823e1a Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Fri, 10 Nov 2023 09:05:29 +0000 Subject: [PATCH 27/33] Update BeekeeperApiIntegrationTest.java Extracting sections as code constants --- .../api/BeekeeperApiIntegrationTest.java | 77 ++++++++++--------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index cdc75cec..1602e2b3 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -67,14 +67,10 @@ public class BeekeeperApiIntegrationTest extends BeekeeperIntegrationTestBase { public ObjectMapper geObjectMapper() { return new ObjectMapper().registerModule(new ParameterNamesModule()).registerModule(new JavaTimeModule()); } - private static final Logger log = LoggerFactory.getLogger(BeekeeperApiIntegrationTest.class); - protected static ConfigurableApplicationContext context; protected BeekeeperApiTestClient testClient; - protected final ObjectMapper mapper = geObjectMapper(); - private Long id = 1L; protected static final String someTable = "some_table"; protected static final String someDatabase = "some_database"; @@ -86,7 +82,14 @@ public ObjectMapper geObjectMapper() { protected static final String partitionA = "event_date=2020-01-01/event_hour=0/event_type=A"; protected static final String partitionB = "event_date=2020-01-01/event_hour=0/event_type=B"; protected static final String partitionC = "event_date=2020-01-01/event_hour=0/event_type=C"; - + protected final HousekeepingPath testPathA = createHousekeepingPath(someTable, pathA, LifecycleEventType.EXPIRED, duration.toString(), HousekeepingStatus.FAILED); + protected final HousekeepingPath testPathB = createHousekeepingPath(someTable, pathB, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.FAILED); + protected final HousekeepingPath testPathC = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.FAILED); + protected final HousekeepingMetadata testMetadataA = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, duration.toString()); + protected final HousekeepingMetadata testMetadataB = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.EXPIRED, duration.toString()); + protected final HousekeepingMetadata testMetadataC = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.EXPIRED, duration.toString()); + protected final HousekeepingMetadata testMetadataD = createHousekeepingMetadata(someTable, pathC, partitionB, LifecycleEventType.UNREFERENCED, duration.toString()); + protected final HousekeepingMetadata testMetadataE = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.UNREFERENCED, duration.toString()); @BeforeEach public void beforeEach() { int port = SocketUtils.findAvailableTcpPort(); @@ -109,10 +112,12 @@ public final void afterEach() { @Test public void testGetMetadataWhenTableNotFoundReturnsEmptyList() throws SQLException, InterruptedException, IOException { - HousekeepingMetadata testMetadata1 = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, duration.toString()); - HousekeepingMetadata testMetadata2 = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.EXPIRED, duration.toString()); - insertExpiredMetadata(testMetadata1); - insertExpiredMetadata(testMetadata2); + HousekeepingMetadata testMetadata1 = testMetadataB; + HousekeepingMetadata testMetadata2 = testMetadataC; + + for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadata1, testMetadata2)) { + insertExpiredMetadata(testMetadata); + } HttpResponse response = testClient.getMetadata("wrong_database", "wrong_table"); assertThat(response.statusCode()).isEqualTo(OK.value()); @@ -124,11 +129,11 @@ public void testGetMetadataWhenTableNotFoundReturnsEmptyList() @Test public void testGetMetadataWhenThereIsFiltering() throws SQLException, InterruptedException, IOException { - HousekeepingMetadata testMetadata1 = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, duration.toString()); - HousekeepingMetadata testMetadata2 = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.UNREFERENCED, duration.toString()); - HousekeepingMetadata testMetadata3 = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.UNREFERENCED, duration.toString()); + HousekeepingMetadata testMetadata1 = testMetadataA; + HousekeepingMetadata testMetadata2 = testMetadataD; + HousekeepingMetadata testMetadata3 = testMetadataE; - testMetadata1.setHousekeepingStatus(HousekeepingStatus.FAILED); +// testMetadata1.setHousekeepingStatus(HousekeepingStatus.FAILED); testMetadata1.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); testMetadata1.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); @@ -156,10 +161,11 @@ public void testGetMetadataWhenThereIsFiltering() throws SQLException, Interrupt @Test public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedException, IOException { - HousekeepingPath testPath1 = createHousekeepingPath(someTable, pathA, LifecycleEventType.EXPIRED, duration.toString()); - HousekeepingPath testPath2 = createHousekeepingPath(someTable, pathB, LifecycleEventType.UNREFERENCED, duration.toString()); - HousekeepingPath testPath3 = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, duration.toString()); - testPath1.setHousekeepingStatus(HousekeepingStatus.FAILED); + HousekeepingPath testPath1 = testPathA; + HousekeepingPath testPath2 = testPathB; + HousekeepingPath testPath3 = testPathC; + +// testPath1.setHousekeepingStatus(HousekeepingStatus.FAILED); testPath1.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); testPath1.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); @@ -185,10 +191,10 @@ public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedE @Test public void testPathsPageable() throws SQLException, InterruptedException, IOException { - - HousekeepingPath testPath1 = createHousekeepingPath(someTable, pathA, LifecycleEventType.EXPIRED, duration.toString()); - HousekeepingPath testPath2 = createHousekeepingPath(someTable, pathB, LifecycleEventType.UNREFERENCED, duration.toString()); - HousekeepingPath testPath3 = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, duration.toString()); + HousekeepingPath testPath1 = testPathA; + HousekeepingPath testPath2 = testPathB; + HousekeepingPath testPath3 = testPathC; +// HousekeepingPath testPath3 = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.DISABLED); for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { testPath.setHousekeepingStatus(HousekeepingStatus.FAILED); @@ -213,9 +219,9 @@ public void testPathsPageable() throws SQLException, InterruptedException, IOExc @Test public void testMetadataPageable() throws SQLException, InterruptedException, IOException { - HousekeepingMetadata testMetadata1 = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, duration.toString()); - HousekeepingMetadata testMetadata2 = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.EXPIRED, duration.toString()); - HousekeepingMetadata testMetadata3 = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.EXPIRED, duration.toString()); + HousekeepingMetadata testMetadata1 = testMetadataA; + HousekeepingMetadata testMetadata2 = testMetadataB; + HousekeepingMetadata testMetadata3 = testMetadataC; for (HousekeepingMetadata testPath : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { testPath.setHousekeepingStatus(HousekeepingStatus.FAILED); @@ -242,22 +248,16 @@ public void testMetadataPageable() throws SQLException, InterruptedException, IO @Disabled @Test public void manualTest() throws SQLException, InterruptedException { - HousekeepingMetadata testMetadata1 = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, - duration.toString()); - HousekeepingMetadata testMetadata2 = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.EXPIRED, - duration.toString()); - HousekeepingMetadata testMetadata3 = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.UNREFERENCED, - duration.toString()); + HousekeepingMetadata testMetadata1 = testMetadataA; + HousekeepingMetadata testMetadata2 = testMetadataB; + HousekeepingMetadata testMetadata3 = testMetadataC; for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { insertExpiredMetadata(testMetadata); } - HousekeepingPath testPath1 = createHousekeepingPath(someTable, pathA, LifecycleEventType.EXPIRED, - duration.toString()); - HousekeepingPath testPath2 = createHousekeepingPath(someTable, pathB, LifecycleEventType.UNREFERENCED, - duration.toString()); - HousekeepingPath testPath3 = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, - duration.toString()); + HousekeepingPath testPath1 = testPathA; + HousekeepingPath testPath2 = testPathB; + HousekeepingPath testPath3 = testPathC; for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { insertUnreferencedPath(testPath); } @@ -318,7 +318,8 @@ private HousekeepingPath createHousekeepingPath( String tableName, String path, LifecycleEventType lifecycleEventType, - String cleanupDelay) { + String cleanupDelay, + HousekeepingStatus housekeepingStatus) { return HousekeepingPath .builder() .id(id++) @@ -335,4 +336,4 @@ private HousekeepingPath createHousekeepingPath( .build(); } -} +} \ No newline at end of file From fdabfc54a11a1f33973921c9cc20c50cea391b7e Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Fri, 10 Nov 2023 09:09:14 +0000 Subject: [PATCH 28/33] Update BeekeeperApiIntegrationTest.java --- .../beekeeper/integration/api/BeekeeperApiIntegrationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index 1602e2b3..f37d50a0 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -194,7 +194,6 @@ public void testPathsPageable() throws SQLException, InterruptedException, IOExc HousekeepingPath testPath1 = testPathA; HousekeepingPath testPath2 = testPathB; HousekeepingPath testPath3 = testPathC; -// HousekeepingPath testPath3 = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.DISABLED); for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { testPath.setHousekeepingStatus(HousekeepingStatus.FAILED); From 9e3f1d3450c73e70c87fea2f3df56343c9afcf90 Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Fri, 10 Nov 2023 09:20:19 +0000 Subject: [PATCH 29/33] Update BeekeeperApiIntegrationTest.java Remove comment from Integration test --- .../beekeeper/integration/api/BeekeeperApiIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index f37d50a0..991c7007 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -165,7 +165,7 @@ public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedE HousekeepingPath testPath2 = testPathB; HousekeepingPath testPath3 = testPathC; -// testPath1.setHousekeepingStatus(HousekeepingStatus.FAILED); + testPath1.setHousekeepingStatus(HousekeepingStatus.FAILED); testPath1.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); testPath1.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); From fd4d4614125ea968e98a9ca941190dfb35380720 Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Fri, 10 Nov 2023 09:21:55 +0000 Subject: [PATCH 30/33] Update BeekeeperApiIntegrationTest.java Remove comment from Integration test --- .../beekeeper/integration/api/BeekeeperApiIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index 991c7007..3b8e8aae 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -133,7 +133,7 @@ public void testGetMetadataWhenThereIsFiltering() throws SQLException, Interrupt HousekeepingMetadata testMetadata2 = testMetadataD; HousekeepingMetadata testMetadata3 = testMetadataE; -// testMetadata1.setHousekeepingStatus(HousekeepingStatus.FAILED); + testMetadata1.setHousekeepingStatus(HousekeepingStatus.FAILED); testMetadata1.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); testMetadata1.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); From c067716502581bc7e38b126a1e76904f5772451b Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Fri, 10 Nov 2023 11:51:39 +0000 Subject: [PATCH 31/33] Refactoring tests --- .../api/BeekeeperApiIntegrationTest.java | 56 ++++++------------- 1 file changed, 17 insertions(+), 39 deletions(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index 3b8e8aae..e848f481 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -112,10 +112,8 @@ public final void afterEach() { @Test public void testGetMetadataWhenTableNotFoundReturnsEmptyList() throws SQLException, InterruptedException, IOException { - HousekeepingMetadata testMetadata1 = testMetadataB; - HousekeepingMetadata testMetadata2 = testMetadataC; - for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadata1, testMetadata2)) { + for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadataB, testMetadataC)) { insertExpiredMetadata(testMetadata); } @@ -129,15 +127,11 @@ public void testGetMetadataWhenTableNotFoundReturnsEmptyList() @Test public void testGetMetadataWhenThereIsFiltering() throws SQLException, InterruptedException, IOException { - HousekeepingMetadata testMetadata1 = testMetadataA; - HousekeepingMetadata testMetadata2 = testMetadataD; - HousekeepingMetadata testMetadata3 = testMetadataE; + testMetadataA.setHousekeepingStatus(HousekeepingStatus.FAILED); + testMetadataA.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); + testMetadataA.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - testMetadata1.setHousekeepingStatus(HousekeepingStatus.FAILED); - testMetadata1.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - testMetadata1.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - - for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { + for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadataA, testMetadataD, testMetadataE)) { insertExpiredMetadata(testMetadata); } @@ -155,21 +149,17 @@ public void testGetMetadataWhenThereIsFiltering() throws SQLException, Interrupt .readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); - assertThatMetadataEqualsResponse(testMetadata1, result.get(0)); + assertThatMetadataEqualsResponse(testMetadataA, result.get(0)); assertThat(result).hasSize(1); } @Test public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedException, IOException { - HousekeepingPath testPath1 = testPathA; - HousekeepingPath testPath2 = testPathB; - HousekeepingPath testPath3 = testPathC; - - testPath1.setHousekeepingStatus(HousekeepingStatus.FAILED); - testPath1.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - testPath1.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); + testPathA.setHousekeepingStatus(HousekeepingStatus.FAILED); + testPathA.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); + testPathA.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); - for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { + for (HousekeepingPath testPath : Arrays.asList(testPathA, testPathB, testPathC)) { insertUnreferencedPath(testPath); } String filters = "?housekeeping_status=FAILED" @@ -185,21 +175,18 @@ public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedE .readValue(body, new TypeReference>() {}); List result = responsePage.getContent(); assertThat(responsePage.getTotalElements()).isEqualTo(1L); - assertThatPathsEqualsResponse(testPath1, result.get(0)); + assertThatPathsEqualsResponse(testPathA, result.get(0)); assertThat(result).hasSize(1); } @Test public void testPathsPageable() throws SQLException, InterruptedException, IOException { - HousekeepingPath testPath1 = testPathA; - HousekeepingPath testPath2 = testPathB; - HousekeepingPath testPath3 = testPathC; - for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { + for (HousekeepingPath testPath : Arrays.asList(testPathA, testPathB, testPathC)) { testPath.setHousekeepingStatus(HousekeepingStatus.FAILED); } - for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { + for (HousekeepingPath testPath : Arrays.asList(testPathA, testPathB, testPathC)) { insertUnreferencedPath(testPath); } @@ -218,15 +205,12 @@ public void testPathsPageable() throws SQLException, InterruptedException, IOExc @Test public void testMetadataPageable() throws SQLException, InterruptedException, IOException { - HousekeepingMetadata testMetadata1 = testMetadataA; - HousekeepingMetadata testMetadata2 = testMetadataB; - HousekeepingMetadata testMetadata3 = testMetadataC; - for (HousekeepingMetadata testPath : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { + for (HousekeepingMetadata testPath : Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { testPath.setHousekeepingStatus(HousekeepingStatus.FAILED); } - for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { + for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { insertExpiredMetadata(testMetadata); } @@ -247,17 +231,11 @@ public void testMetadataPageable() throws SQLException, InterruptedException, IO @Disabled @Test public void manualTest() throws SQLException, InterruptedException { - HousekeepingMetadata testMetadata1 = testMetadataA; - HousekeepingMetadata testMetadata2 = testMetadataB; - HousekeepingMetadata testMetadata3 = testMetadataC; - for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadata1, testMetadata2, testMetadata3)) { + for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { insertExpiredMetadata(testMetadata); } - HousekeepingPath testPath1 = testPathA; - HousekeepingPath testPath2 = testPathB; - HousekeepingPath testPath3 = testPathC; - for (HousekeepingPath testPath : Arrays.asList(testPath1, testPath2, testPath3)) { + for (HousekeepingPath testPath : Arrays.asList(testPathA, testPathB, testPathC)) { insertUnreferencedPath(testPath); } From 6d5e33d616040a91c3addc3836668db17232da7b Mon Sep 17 00:00:00 2001 From: Hamza Jugon <104994559+HamzaJugon@users.noreply.github.com> Date: Fri, 10 Nov 2023 11:53:22 +0000 Subject: [PATCH 32/33] Update beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java Co-authored-by: Andreea Paduraru <40387422+andreeapad@users.noreply.github.com> --- .../beekeeper/integration/api/BeekeeperApiIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index e848f481..f2d73e67 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -313,4 +313,4 @@ private HousekeepingPath createHousekeepingPath( .build(); } -} \ No newline at end of file +} From d853eb73f6085006e7f94ab0f9b3218e6fc47be9 Mon Sep 17 00:00:00 2001 From: Hamza Jugon Date: Fri, 10 Nov 2023 15:09:39 +0000 Subject: [PATCH 33/33] Refactoring tests --- .../api/BeekeeperApiIntegrationTest.java | 29 ++++++------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java index f2d73e67..c311dde0 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/api/BeekeeperApiIntegrationTest.java @@ -85,11 +85,11 @@ public ObjectMapper geObjectMapper() { protected final HousekeepingPath testPathA = createHousekeepingPath(someTable, pathA, LifecycleEventType.EXPIRED, duration.toString(), HousekeepingStatus.FAILED); protected final HousekeepingPath testPathB = createHousekeepingPath(someTable, pathB, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.FAILED); protected final HousekeepingPath testPathC = createHousekeepingPath(someTable, pathC, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.FAILED); - protected final HousekeepingMetadata testMetadataA = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, duration.toString()); - protected final HousekeepingMetadata testMetadataB = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.EXPIRED, duration.toString()); - protected final HousekeepingMetadata testMetadataC = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.EXPIRED, duration.toString()); - protected final HousekeepingMetadata testMetadataD = createHousekeepingMetadata(someTable, pathC, partitionB, LifecycleEventType.UNREFERENCED, duration.toString()); - protected final HousekeepingMetadata testMetadataE = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.UNREFERENCED, duration.toString()); + protected final HousekeepingMetadata testMetadataA = createHousekeepingMetadata(someTable, pathA, partitionA, LifecycleEventType.EXPIRED, duration.toString(),HousekeepingStatus.FAILED); + protected final HousekeepingMetadata testMetadataB = createHousekeepingMetadata(someTable, pathB, partitionB, LifecycleEventType.EXPIRED, duration.toString(),HousekeepingStatus.FAILED); + protected final HousekeepingMetadata testMetadataC = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.EXPIRED, duration.toString(),HousekeepingStatus.FAILED); + protected final HousekeepingMetadata testMetadataD = createHousekeepingMetadata(someTable, pathC, partitionB, LifecycleEventType.UNREFERENCED, duration.toString(), HousekeepingStatus.SCHEDULED); + protected final HousekeepingMetadata testMetadataE = createHousekeepingMetadata(someTable, pathC, partitionC, LifecycleEventType.UNREFERENCED, duration.toString(),HousekeepingStatus.SCHEDULED); @BeforeEach public void beforeEach() { int port = SocketUtils.findAvailableTcpPort(); @@ -127,7 +127,6 @@ public void testGetMetadataWhenTableNotFoundReturnsEmptyList() @Test public void testGetMetadataWhenThereIsFiltering() throws SQLException, InterruptedException, IOException { - testMetadataA.setHousekeepingStatus(HousekeepingStatus.FAILED); testMetadataA.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); testMetadataA.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); @@ -155,7 +154,6 @@ public void testGetMetadataWhenThereIsFiltering() throws SQLException, Interrupt @Test public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedException, IOException { - testPathA.setHousekeepingStatus(HousekeepingStatus.FAILED); testPathA.setCleanupTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); testPathA.setCreationTimestamp(LocalDateTime.parse("1999-05-05T10:41:20")); @@ -181,11 +179,6 @@ public void testGetPathsWhenThereIsFiltering() throws SQLException, InterruptedE @Test public void testPathsPageable() throws SQLException, InterruptedException, IOException { - - for (HousekeepingPath testPath : Arrays.asList(testPathA, testPathB, testPathC)) { - testPath.setHousekeepingStatus(HousekeepingStatus.FAILED); - } - for (HousekeepingPath testPath : Arrays.asList(testPathA, testPathB, testPathC)) { insertUnreferencedPath(testPath); } @@ -205,11 +198,6 @@ public void testPathsPageable() throws SQLException, InterruptedException, IOExc @Test public void testMetadataPageable() throws SQLException, InterruptedException, IOException { - - for (HousekeepingMetadata testPath : Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { - testPath.setHousekeepingStatus(HousekeepingStatus.FAILED); - } - for (HousekeepingMetadata testMetadata : Arrays.asList(testMetadataA, testMetadataB, testMetadataC)) { insertExpiredMetadata(testMetadata); } @@ -273,7 +261,8 @@ private HousekeepingMetadata createHousekeepingMetadata( String path, String partitionName, LifecycleEventType lifecycleEventType, - String cleanupDelay) { + String cleanupDelay, + HousekeepingStatus housekeepingStatus) { return HousekeepingMetadata .builder() .id(id++) @@ -281,7 +270,7 @@ private HousekeepingMetadata createHousekeepingMetadata( .databaseName(DATABASE_NAME_VALUE) .tableName(tableName) .partitionName(partitionName) - .housekeepingStatus(SCHEDULED) + .housekeepingStatus(housekeepingStatus) .creationTimestamp(CREATION_TIMESTAMP_VALUE) .modifiedTimestamp(CREATION_TIMESTAMP_VALUE) .cleanupDelay(PeriodDuration.parse(cleanupDelay)) @@ -303,7 +292,7 @@ private HousekeepingPath createHousekeepingPath( .path(path) .databaseName(DATABASE_NAME_VALUE) .tableName(tableName) - .housekeepingStatus(SCHEDULED) + .housekeepingStatus(housekeepingStatus) .creationTimestamp(CREATION_TIMESTAMP_VALUE) .modifiedTimestamp(CREATION_TIMESTAMP_VALUE) .cleanupDelay(PeriodDuration.parse(cleanupDelay))