Skip to content

Commit

Permalink
Add check to ensure file index is used correctly (#333)
Browse files Browse the repository at this point in the history
# Fixes
- Add check for when `MixedLogReader` is rewound in order to prevent a
numerical overflow error
- Set default next index element when file is being read from beginning
  • Loading branch information
wbrannon authored Aug 14, 2024
2 parents 702e065 + 57558ee commit 29786ba
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion python/fusion_engine_client/parsers/mixed_log_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,9 @@ def _print_progress(self, file_size=None):
if file_size is None:
file_size = min(self.file_size_bytes, self.max_bytes)

if self.total_bytes_read - self.last_print_bytes > 10e6 or self.total_bytes_read == file_size:
if self.total_bytes_read < self.last_print_bytes or \
self.total_bytes_read - self.last_print_bytes > 10e6 or \
self.total_bytes_read == file_size:
elapsed_sec = (datetime.now() - self.start_time).total_seconds()
self.logger.log(logging.INFO if show_progress else logging.DEBUG,
'Processed %d/%d bytes (%.1f%%). [elapsed=%.1f sec, rate=%.1f MB/s]' %
Expand Down Expand Up @@ -589,6 +591,8 @@ def filter_in_place(self, key, clear_existing: Union[bool, str] = False,
if self.index is not None:
if len(self.index) == 0:
self.next_index_elem = 0
elif prev_offset_bytes < 0:
self.next_index_elem = 0
else:
idx = np.argmax(self.index.offset > prev_offset_bytes)
if idx == 0 and self.index.offset[0] <= prev_offset_bytes:
Expand Down

0 comments on commit 29786ba

Please sign in to comment.