From 212328c740acd66c0940495e00ac0a67d6919589 Mon Sep 17 00:00:00 2001 From: Michael Dombrowski Date: Thu, 16 Nov 2023 10:14:35 -0500 Subject: [PATCH 1/2] test(uat): log rotation with minimum log level setting --- .../greengrass/logmanager/model/LogFileGroup.java | 2 +- uat/codebuild/uat_linux_buildspec.yaml | 2 +- .../java/com/aws/greengrass/steps/FileSteps.java | 5 +++-- .../greengrass/features/log-manager-1.feature | 12 ++++++++---- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/aws/greengrass/logmanager/model/LogFileGroup.java b/src/main/java/com/aws/greengrass/logmanager/model/LogFileGroup.java index a263d6f6..2e87160f 100644 --- a/src/main/java/com/aws/greengrass/logmanager/model/LogFileGroup.java +++ b/src/main/java/com/aws/greengrass/logmanager/model/LogFileGroup.java @@ -234,7 +234,7 @@ public long totalSizeInBytes() { public boolean hasExceededDiskUsage() { return this.maxBytes .map((val) -> this.totalSizeInBytes() > val) - .orElseGet(() -> false); + .orElse(false); } /** diff --git a/uat/codebuild/uat_linux_buildspec.yaml b/uat/codebuild/uat_linux_buildspec.yaml index e8894f19..ec15b0c2 100644 --- a/uat/codebuild/uat_linux_buildspec.yaml +++ b/uat/codebuild/uat_linux_buildspec.yaml @@ -14,7 +14,7 @@ phases: - mvn -U -ntp verify -DskipTests=true - mvn -U -ntp clean verify -f uat/pom.xml - java -Dggc.archive=/tmp/greengrass-nucleus-latest.zip - -Dtags='LogManager&!unstable' -Dggc.install.root=$CODEBUILD_SRC_DIR -Dggc.log.level=INFO -Daws.region=us-west-2 + -Dtags='MyTest&LogManager&!unstable' -Dggc.install.root=$CODEBUILD_SRC_DIR -Dggc.log.level=INFO -Daws.region=us-west-2 -jar uat/testing-features/target/greengrass-log-manager-testing-features.jar artifacts: diff --git a/uat/testing-features/src/main/java/com/aws/greengrass/steps/FileSteps.java b/uat/testing-features/src/main/java/com/aws/greengrass/steps/FileSteps.java index 61cbc0c4..6e3d176c 100644 --- a/uat/testing-features/src/main/java/com/aws/greengrass/steps/FileSteps.java +++ b/uat/testing-features/src/main/java/com/aws/greengrass/steps/FileSteps.java @@ -64,7 +64,8 @@ private static List generateRandomMessages(int n, int length) { List msgs = new ArrayList<>(); for (int i = 0; i < n; i++) { // TODO: Improves this as this is not how the logger writes the logs - msgs.add(RANDOM_STRING_GENERATOR.generate(length)); + String message = RANDOM_STRING_GENERATOR.generate(length); + msgs.add("{\"level\": \"INFO\", \"message\": \"" + message + "\"}"); } return msgs; } @@ -168,4 +169,4 @@ public void verifyActiveFile(String componentName, String directoryAlias) { List sortedFileList = getComponentLogFiles(componentName, logsDirectory); assertEquals(1, sortedFileList.size()); } -} \ No newline at end of file +} diff --git a/uat/testing-features/src/main/resources/greengrass/features/log-manager-1.feature b/uat/testing-features/src/main/resources/greengrass/features/log-manager-1.feature index b0d3230e..bc7597fc 100644 --- a/uat/testing-features/src/main/resources/greengrass/features/log-manager-1.feature +++ b/uat/testing-features/src/main/resources/greengrass/features/log-manager-1.feature @@ -148,9 +148,10 @@ Feature: Greengrass V2 LogManager And I verify 100 logs for UserComponentX of type UserComponent have been uploaded to Cloudwatch within 120 seconds And I verify the rotated files are deleted and that the active log file is present for component UserComponentX on directory UserComponentXLogDirectory + @MyTest Scenario: LogManager-1-T3: As a customer I can configure the logs uploader to delete log oldest log files to keep the disk space limit specified on the configuration - Given 10 temporary rotated log files for component UserComponentB have been created + Given 15 temporary rotated log files for component UserComponentB have been created And 5 temporary rotated log files for component aws.greengrass.Nucleus have been created And 5 temporary rotated log files for component UserComponentA have been created Given I create a Greengrass deployment with components @@ -166,7 +167,10 @@ Feature: Greengrass V2 LogManager "logFileRegex": "UserComponentB_(.)+.log", "logFileDirectoryPath": "${UserComponentBLogDirectory}", "diskSpaceLimit":"100", - "diskSpaceLimitUnit":"KB" + "diskSpaceLimitUnit":"KB", + "minimumLogLevel": "INFO", + "uploadToCloudWatch": "true", + "deleteLogFileAfterCloudUpload": "false" } }, "systemLogsConfiguration": { @@ -177,7 +181,7 @@ Feature: Greengrass V2 LogManager "deleteLogFileAfterCloudUpload": "true" } }, - "periodicUploadIntervalSec": "500" + "periodicUploadIntervalSec": "0.1" } } """ @@ -185,7 +189,7 @@ Feature: Greengrass V2 LogManager Then the Greengrass deployment is COMPLETED on the device after 3 minutes Then I verify the aws.greengrass.LogManager component is RUNNING using the greengrass-cli And I wait 5 seconds - Then I verify that 10 log files for component UserComponentB are still available + Then I verify that 15 log files for component UserComponentB are still available @network Scenario: LogManager-1-T4: As a developer, logs uploader will handle network interruptions gracefully and upload logs from the last uploaded log after network resumes From bcd235e27558b00d0f27fa8181acb7c13c31f2db Mon Sep 17 00:00:00 2001 From: Michael Dombrowski Date: Thu, 16 Nov 2023 10:26:15 -0500 Subject: [PATCH 2/2] fix: delete log even if nothing was uploaded --- .../greengrass/logmanager/model/LogFileGroup.java | 2 +- .../greengrass/features/log-manager-1.feature | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/aws/greengrass/logmanager/model/LogFileGroup.java b/src/main/java/com/aws/greengrass/logmanager/model/LogFileGroup.java index 2e87160f..a263d6f6 100644 --- a/src/main/java/com/aws/greengrass/logmanager/model/LogFileGroup.java +++ b/src/main/java/com/aws/greengrass/logmanager/model/LogFileGroup.java @@ -234,7 +234,7 @@ public long totalSizeInBytes() { public boolean hasExceededDiskUsage() { return this.maxBytes .map((val) -> this.totalSizeInBytes() > val) - .orElse(false); + .orElseGet(() -> false); } /** diff --git a/uat/testing-features/src/main/resources/greengrass/features/log-manager-1.feature b/uat/testing-features/src/main/resources/greengrass/features/log-manager-1.feature index bc7597fc..71490b90 100644 --- a/uat/testing-features/src/main/resources/greengrass/features/log-manager-1.feature +++ b/uat/testing-features/src/main/resources/greengrass/features/log-manager-1.feature @@ -152,6 +152,7 @@ Feature: Greengrass V2 LogManager Scenario: LogManager-1-T3: As a customer I can configure the logs uploader to delete log oldest log files to keep the disk space limit specified on the configuration Given 15 temporary rotated log files for component UserComponentB have been created + And 15 temporary rotated log files for component UserComponentC have been created And 5 temporary rotated log files for component aws.greengrass.Nucleus have been created And 5 temporary rotated log files for component UserComponentA have been created Given I create a Greengrass deployment with components @@ -168,6 +169,15 @@ Feature: Greengrass V2 LogManager "logFileDirectoryPath": "${UserComponentBLogDirectory}", "diskSpaceLimit":"100", "diskSpaceLimitUnit":"KB", + "minimumLogLevel": "WARN", + "uploadToCloudWatch": "true", + "deleteLogFileAfterCloudUpload": "false" + }, + "UserComponentC": { + "logFileRegex": "UserComponentC_(.)+.log", + "logFileDirectoryPath": "${UserComponentCLogDirectory}", + "diskSpaceLimit":"100", + "diskSpaceLimitUnit":"KB", "minimumLogLevel": "INFO", "uploadToCloudWatch": "true", "deleteLogFileAfterCloudUpload": "false" @@ -189,7 +199,8 @@ Feature: Greengrass V2 LogManager Then the Greengrass deployment is COMPLETED on the device after 3 minutes Then I verify the aws.greengrass.LogManager component is RUNNING using the greengrass-cli And I wait 5 seconds - Then I verify that 15 log files for component UserComponentB are still available + Then I verify that 9 log files for component UserComponentB are still available + And I verify that 9 log files for component UserComponentC are still available @network Scenario: LogManager-1-T4: As a developer, logs uploader will handle network interruptions gracefully and upload logs from the last uploaded log after network resumes