Skip to content

Commit

Permalink
fixing timeout on postmans (#30331)
Browse files Browse the repository at this point in the history
### Proposed Changes
* This should fix a timeout in our Postman test reading a large file
  • Loading branch information
fabrizzio-dotCMS authored Oct 11, 2024
1 parent 9a1866f commit 183afba
Showing 1 changed file with 47 additions and 16 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.jetbrains.annotations.Nullable;

/**
* This class reads a large file and prints the content to the log.
Expand All @@ -26,6 +25,8 @@
@Queue("demo")
public class LargeFileReader implements JobProcessor, Cancellable {

public static final int LOG_EVERY_LINES = 1;
public static final int DEFAULT_MAX_LINES = 2000;
private boolean working = true;

@Override
Expand All @@ -35,31 +36,25 @@ public void process(Job job) {
Logger.info(this.getClass(), "Processing job: " + job.id());
Map<String, Object> params = job.parameters();

final Optional<Integer> linesParam = linesParam(params);
if (linesParam.isEmpty()) {
Logger.error(this.getClass(),
"Unable to retrieve the number of lines to read. Quitting the job.");
throw new DotRuntimeException("Unable to retrieve the temporary file.");
}

Optional<DotTempFile> tempFile = tempFile(params);
if (tempFile.isEmpty()) {
Logger.error(this.getClass(), "Unable to retrieve the temporary file. Quitting the job.");
throw new DotRuntimeException("Unable to retrieve the temporary file.");
}

final int nLines = linesParam.get();
final int nLines = linesParam(params).orElse(LOG_EVERY_LINES);
final int maxLines = maxLinesParam(params).orElse(DEFAULT_MAX_LINES);
final DotTempFile dotTempFile = tempFile.get();

doReadLargeFile(dotTempFile, nLines, job);
doReadLargeFile(dotTempFile, nLines, maxLines, job);
}

/**
* Process the job
* @param dotTempFile temporary file
* @param nLines number of lines to read and print
*/
private void doReadLargeFile(DotTempFile dotTempFile, int nLines, final Job job) {
private void doReadLargeFile(DotTempFile dotTempFile, int nLines, int maxLines ,final Job job) {
final Long totalCount = countLines(dotTempFile);
if (totalCount == null) {
return;
Expand All @@ -82,11 +77,15 @@ private void doReadLargeFile(DotTempFile dotTempFile, int nLines, final Job job)
// Print the line when the counter reaches nLines
if (lineCount == nLines) {
lineCount = 0; // Reset the counter
Logger.info(this.getClass(), line);
Logger.debug(this.getClass(), line);
delay();
}
final float progressPercentage = ((float) readCount / totalCount);
progressTracker.ifPresent(tracker -> tracker.updateProgress(progressPercentage));
if (readCount >= maxLines) {
Logger.info(this.getClass(), "Max lines reached. Stopping the job.");
break;
}
}

Logger.info(this.getClass(), "Reading completed. Total lines read: " + readCount);
Expand All @@ -96,17 +95,20 @@ private void doReadLargeFile(DotTempFile dotTempFile, int nLines, final Job job)
}
}

private @Nullable Long countLines(DotTempFile dotTempFile) {
/**
* Count the number of lines in the file
* @param dotTempFile temporary file
* @return the number of lines in the file
*/
private Long countLines(DotTempFile dotTempFile) {
long totalCount = 0;
try (BufferedReader reader = new BufferedReader(new FileReader(dotTempFile.file))) {
totalCount = reader.lines().count();
if (totalCount == 0) {
Logger.info(this.getClass(), "No lines in the file: " + dotTempFile.file.getName());
return null;
}
} catch (Exception e) {
Logger.error(this.getClass(), "Unexpected error during processing: " + e.getMessage());
return null;
}
return totalCount;
}
Expand All @@ -118,6 +120,35 @@ private void delay() {
}).onFailure(e->Logger.error(this.getClass(), "Error during delay", e));
}

/**
* Retrieve the maximum number of lines to read from the parameters
*
* @param params input parameters
* @return the maximum number of lines to read
*/
Optional<Integer> maxLinesParam(Map<String, Object> params) {
final Object maxLinesRaw = params.get("maxLines");
if (!(maxLinesRaw instanceof String)) {
return Optional.empty();
}

int maxLines;
try {
maxLines = Integer.parseInt((String) maxLinesRaw);
} catch (NumberFormatException e) {
Logger.error(this.getClass(), "Parameter 'maxLines' must be a valid integer.", e);
return Optional.empty();
}

// Validate required parameters
if (maxLines <= 0) {
Logger.error(this.getClass(), "Parameters 'maxLines' is required.");
return Optional.empty();
}

return Optional.of(maxLines);
}

/**
* Retrieve the number of lines to read from the parameters
*
Expand All @@ -142,7 +173,7 @@ Optional<Integer> linesParam(Map<String, Object> params) {

// Validate required parameters
if (nLines <= 0) {
Logger.error(this.getClass(), "Parameters 'tempFileId' are required.");
Logger.error(this.getClass(), "Parameters 'nLines' is required.");
return Optional.empty();
}

Expand Down

0 comments on commit 183afba

Please sign in to comment.