Skip to content

Commit

Permalink
show load/extract rates in minutes and seconds
Browse files Browse the repository at this point in the history
Showing long-running load/extract rates in minutes and seconds instead of seconds only to make it easier to read.
  • Loading branch information
ashitsalesforce committed Jan 19, 2024
1 parent d9e465b commit 88ff207
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 17 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ public String calculateSubTask(long processedRecordsInJob, long numErrorsInJob)
long hourlyProcessingRate;

final long totalElapsedTimeInSec = (currentLoadTime.getTime() - this.startTime.getTime())/1000;
final long elapsedTimeInMinutes = totalElapsedTimeInSec / 60;
if (totalElapsedTimeInSec == 0) {
hourlyProcessingRate = 0;
} else {
Expand All @@ -89,7 +90,7 @@ public String calculateSubTask(long processedRecordsInJob, long numErrorsInJob)
numSuccessInJob, // {2}
numErrorsInJob); // {3}
}
// LoadRateCalculator.processed=Processed {0} of {1} total records in {7} seconds.
// LoadRateCalculator.processed=Processed {0} of {1} total records in {8} minutes, {7} seconds.
// There are {5} successes and {6} errors. \nRate: {2} records per hour.
// Estimated time to complete: {3} minutes and {4} seconds.
return Messages.getMessage(getClass(), "processed",
Expand All @@ -100,7 +101,8 @@ public String calculateSubTask(long processedRecordsInJob, long numErrorsInJob)
remainingSeconds, // {4}
numSuccessInJob, // {5}
numErrorsInJob, // {6}
totalElapsedTimeInSec // {7}
totalElapsedTimeInSec - (60 * elapsedTimeInMinutes), // {7}
elapsedTimeInMinutes // {8}
);
}
}
4 changes: 2 additions & 2 deletions src/main/resources/messages.properties
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ AbstractExtractAction.errorMissingFrom=Error in query: missing [ from ]: {0}
AbstractExtractAction.errorInvalidFieldName=Error in query: invalid field name [{0}] in query string: {1}
AbstractExtractAction.errorMalformedQuery=Error in query: Malformed query string: {0}
AbstractExtractAction.errorEmptyQuery=Error in query: query is empty
LoadRateCalculator.processed=Processed {0} of {1} total records in {7} seconds. There are {5} successes and {6} errors. \nRate: {2} records per hour. Estimated time to complete: {3} minutes and {4} seconds.
LoadRateCalculator.processedTimeUnknown=Processed {0} of {1} total records. There are {2} successes and {3} errors.
LoadRateCalculator.processed=Processed {0} of {1} records in {8} minutes, {7} seconds with {5} successes and {6} errors. \nRate: {2} records per hour. Estimated remaining time to complete: {3} minutes and {4} seconds.
LoadRateCalculator.processedTimeUnknown=Processed {0} of {1} records with {2} successes and {3} errors.
Visitor.emptyRowIgnored=Item #: {0} will not be loaded due to the empty input data
Visitor.conversionException=Conversion Exception, writing to errors.csv
Visitor.conversionErrorMsg=Error converting value to correct data type: {0}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public void testRateCalculatorZeroCompletion() {
rateCalculator.start(3600);
String message = rateCalculator.calculateSubTask(0, 0);
assertEquals("incorrect rate calculation: ",
"Processed 0 of 3,600 total records. There are 0 successes and 0 errors.",
"Processed 0 of 3,600 records with 0 successes and 0 errors.",
message);

}
Expand All @@ -55,7 +55,8 @@ public void testRateCalculatorFirstBatchCompletion() {
e.printStackTrace();
}
message = rateCalculator.calculateSubTask(1, 0);
boolean success = message.contains("There are 1 successes and 0 errors. \nRate: 3,600 records per hour.");
boolean success = message.contains("1 successes and 0 errors")
&& message.contains("Rate: 3,600 records per hour");
assertTrue("incorrect rate calculation: ", success);
}

Expand All @@ -79,8 +80,8 @@ public void testRateCalculatorMultiBatchCompletion() {
e.printStackTrace();
}
message = rateCalculator.calculateSubTask(2, 1);
boolean success = message.contains("Processed 2 of 3,600 total records in 2 seconds. There are 1 successes and 1 errors. \nRate: 3,600 records per hour.");
assertTrue("incorrect rate calculation: ", success);
boolean success = message.contains("Processed 2 of 3,600 records in 0 minutes, 2 seconds with 1 successes and 1 errors. \nRate: 3,600 records per hour.");
assertTrue("incorrect rate calculation - " + message, success);

rateCalculator.start(3600);
try {
Expand All @@ -90,8 +91,11 @@ public void testRateCalculatorMultiBatchCompletion() {
e.printStackTrace();
}
message = rateCalculator.calculateSubTask(3, 1);
success = message.contains("Processed 3 of 3,600 total records in 3 seconds. There are 2 successes and 1 errors. \nRate: 3,600 records per hour.");
assertTrue("incorrect rate calculation: ", success);
success = message.contains("Processed 3 of 3,600 records")
&& message.contains("0 minutes, 3 seconds")
&& message.contains("2 successes and 1 errors")
&& message.contains("Rate: 3,600 records per hour");
assertTrue("incorrect rate calculation - " + message, success);

rateCalculator.start(3600);
try {
Expand All @@ -101,19 +105,26 @@ public void testRateCalculatorMultiBatchCompletion() {
e.printStackTrace();
}
message = rateCalculator.calculateSubTask(4, 1);
success = message.contains("Processed 4 of 3,600 total records in 4 seconds. There are 3 successes and 1 errors. \nRate: 3,600 records per hour.");
assertTrue("incorrect rate calculation: ", success);
success = message.contains("Processed 4 of 3,600 records")
&& message.contains("0 minutes, 4 seconds")
&& message.contains("3 successes and 1 errors")
&& message.contains("Rate: 3,600 records per hour");
assertTrue("incorrect rate calculation - " + message, success);

rateCalculator.start(3600);
try {
Thread.sleep(3000);
Thread.sleep(63000); // sleep for a minute
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
message = rateCalculator.calculateSubTask(7, 2);
success = message.contains("Processed 7 of 3,600 total records in 7 seconds. There are 5 successes and 2 errors. \nRate: 3,600 records per hour.");
assertTrue("incorrect rate calculation: ", success);
success = message.contains("Processed 7 of 3,600 records in 1 minutes, 7 seconds with 5 successes and 2 errors. \nRate: 376 records per hour.");
success = message.contains("Processed 7 of 3,600 records")
&& message.contains("1 minutes, 7 seconds")
&& message.contains("5 successes and 2 errors")
&& message.contains("Rate: 376 records per hour");
assertTrue("incorrect rate calculation - " + message, success);
}

@Test
Expand All @@ -129,8 +140,8 @@ public void testRateCalculatorLongTimeForCompletion() {
}
message = rateCalculator.calculateSubTask(1, 0);
assertEquals("incorrect rate calculation: ",
"Processed 1 of 999,999,999 total records. "
+ "There are 1 successes and 0 errors.",
"Processed 1 of 999,999,999 records with 1 successes and 0 errors.",
message);
}

}

0 comments on commit 88ff207

Please sign in to comment.