Skip to content

Commit

Permalink
Feat: Enable logging when new stream starts (#288)
Browse files Browse the repository at this point in the history
  • Loading branch information
vspanxcode authored Jul 8, 2024
1 parent 8676125 commit aea02e1
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 1 deletion.
7 changes: 7 additions & 0 deletions airbyte/sources/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ def __init__(
self.executor = executor
self.name = name
self._processed_records = 0
self._stream_names_observed: set[str] = set()
self._config_dict: dict[str, Any] | None = None
self._last_log_messages: list[str] = []
self._discovered_catalog: AirbyteCatalog | None = None
Expand Down Expand Up @@ -582,6 +583,9 @@ def _execute(self, args: list[str]) -> Iterator[AirbyteMessage]:
message: AirbyteMessage = AirbyteMessage.model_validate_json(json_data=line)
if message.type is Type.RECORD:
self._processed_records += 1
if message.record.stream not in self._stream_names_observed:
self._stream_names_observed.add(message.record.stream)
self._log_stream_read_start(message.record.stream)
if message.type == Type.LOG:
self._add_to_logs(message.log.message)
if message.type == Type.TRACE and message.trace.type == TraceType.ERROR:
Expand Down Expand Up @@ -620,6 +624,9 @@ def _log_sync_start(
event_type=EventType.SYNC,
)

def _log_stream_read_start(self, stream: str) -> None:
print(f"Read started on stream: {stream} at {pendulum.now().format('HH:mm:ss')}...")

def _log_sync_success(
self,
*,
Expand Down
2 changes: 1 addition & 1 deletion tests/unit_tests/test_lowcode_connectors.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,4 @@ def test_nocode_execution(connector_name: str, config: dict) -> None:
source.read()
for name, records in source.read().streams.items():
assert name
assert len(records) > 0
assert len(records) > 0, f"No records were returned from the '{name}' stream."

0 comments on commit aea02e1

Please sign in to comment.