From 98839747509c03d815cc15f704a73a7709c269cb Mon Sep 17 00:00:00 2001 From: William Brannon Date: Wed, 14 Aug 2024 11:53:34 -0600 Subject: [PATCH 1/2] Add check for when reader is rewinded --- python/fusion_engine_client/parsers/mixed_log_reader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/fusion_engine_client/parsers/mixed_log_reader.py b/python/fusion_engine_client/parsers/mixed_log_reader.py index 64b34faa..f6341f26 100644 --- a/python/fusion_engine_client/parsers/mixed_log_reader.py +++ b/python/fusion_engine_client/parsers/mixed_log_reader.py @@ -416,7 +416,9 @@ def _print_progress(self, file_size=None): if file_size is None: file_size = min(self.file_size_bytes, self.max_bytes) - if self.total_bytes_read - self.last_print_bytes > 10e6 or self.total_bytes_read == file_size: + if self.total_bytes_read < self.last_print_bytes or \ + self.total_bytes_read - self.last_print_bytes > 10e6 or \ + self.total_bytes_read == file_size: elapsed_sec = (datetime.now() - self.start_time).total_seconds() self.logger.log(logging.INFO if show_progress else logging.DEBUG, 'Processed %d/%d bytes (%.1f%%). [elapsed=%.1f sec, rate=%.1f MB/s]' % From 57558ee73fe519261235475ec04e53e35cc739cb Mon Sep 17 00:00:00 2001 From: William Brannon Date: Wed, 14 Aug 2024 11:56:48 -0600 Subject: [PATCH 2/2] Set default index element if just starting to read file --- python/fusion_engine_client/parsers/mixed_log_reader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/fusion_engine_client/parsers/mixed_log_reader.py b/python/fusion_engine_client/parsers/mixed_log_reader.py index f6341f26..700117fa 100644 --- a/python/fusion_engine_client/parsers/mixed_log_reader.py +++ b/python/fusion_engine_client/parsers/mixed_log_reader.py @@ -591,6 +591,8 @@ def filter_in_place(self, key, clear_existing: Union[bool, str] = False, if self.index is not None: if len(self.index) == 0: self.next_index_elem = 0 + elif prev_offset_bytes < 0: + self.next_index_elem = 0 else: idx = np.argmax(self.index.offset > prev_offset_bytes) if idx == 0 and self.index.offset[0] <= prev_offset_bytes: