From 351104abd0d1c008a976a577349d23c0ab44d6cd Mon Sep 17 00:00:00 2001 From: Teresa Pelinski Date: Wed, 27 Nov 2024 15:41:15 +0000 Subject: [PATCH 1/5] disconnect and connect refactor --- pybela/Logger.py | 4 +- pybela/Streamer.py | 8 ++- pybela/Watcher.py | 139 +++++++++++++++++++++++++++------------------ test/test.py | 61 +++++++++----------- 4 files changed, 119 insertions(+), 93 deletions(-) diff --git a/pybela/Logger.py b/pybela/Logger.py index 613f195..b4ea32c 100644 --- a/pybela/Logger.py +++ b/pybela/Logger.py @@ -202,8 +202,6 @@ async def async_stop_logging(variables=[]): self.sftp_client.close() - # self.stop() - return asyncio.run(async_stop_logging(variables)) def connect_ssh(self): @@ -579,5 +577,5 @@ async def _async_action_action_on_all_bin_files_in_project(): # FIXME can we av return tasks def __del__(self): + super().__del__() self.disconnect_ssh() # disconnect ssh - self.stop() # stop websockets diff --git a/pybela/Streamer.py b/pybela/Streamer.py index 69ec068..75f2c84 100644 --- a/pybela/Streamer.py +++ b/pybela/Streamer.py @@ -231,8 +231,6 @@ def stop_streaming(self, variables=[]): streaming_buffers_queue (dict): Dict containing the streaming buffers for each streamed variable. """ async def async_stop_streaming(variables=[]): - # self.stop() - _previous_streaming_mode = copy.copy(self._streaming_mode) self._streaming_mode = "OFF" @@ -820,3 +818,9 @@ def _check_periods(self, periods, variables): p, int), "Periods must be integers" return periods + + def __del__(self): + super().__del__() + self._cancel_tasks(tasks=[ + self._on_buffer_callback_worker_task, + self._on_block_callback_worker_task]) diff --git a/pybela/Watcher.py b/pybela/Watcher.py index 88f9599..dc88743 100644 --- a/pybela/Watcher.py +++ b/pybela/Watcher.py @@ -11,7 +11,7 @@ class Watcher: def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add="gui_control"): - """ Watcher class + """ Watcher class - manages websockets and abstracts communication with the Bela watcher Args: ip (str, optional): Remote address IP. If using internet over USB, the IP won't work, pass "bela.local". Defaults to "192.168.7.2". @@ -32,15 +32,16 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add self.ws_ctrl = None self.ws_data = None - self._list_response_queue = asyncio.Queue() + # tasks + self._ctrl_listener_task = None + self._data_listener_task = None + self._process_received_data_msg_task = None + self._send_data_msg_task = None - # receive data message queue + # queues self._received_data_msg_queue = asyncio.Queue() - self._process_received_data_msg_worker_task = None - - # send data message queue + self._list_response_queue = asyncio.Queue() self._to_send_data_msg_queue = asyncio.Queue() - self._sending_data_msg_worker_task = None self._watcher_vars = None @@ -119,7 +120,11 @@ async def _async_connect(): await self._pybela_ws_register[self._mode][self.ws_ctrl_add].close() # Control and monitor can't be used at the same time - if (self._mode == "MONITOR" and self._pybela_ws_register["CONTROL"].get(self.ws_ctrl_add) is not None and self._pybela_ws_register["CONTROL"][self.ws_ctrl_add].open) or (self._mode == "CONTROL" and self._pybela_ws_register["MONITOR"].get(self.ws_ctrl_add) is not None and self._pybela_ws_register["MONITOR"][self.ws_ctrl_add].open): + _is_control_mode_running = self._pybela_ws_register["CONTROL"].get( + self.ws_ctrl_add) is not None and self._pybela_ws_register["CONTROL"][self.ws_ctrl_add].open + _is_monitor_mode_running = self._pybela_ws_register["MONITOR"].get( + self.ws_ctrl_add) is not None and self._pybela_ws_register["MONITOR"][self.ws_ctrl_add].open + if (self._mode == "MONITOR" and _is_control_mode_running) or (self._mode == "CONTROL" and _is_monitor_mode_running): _print_warning( f"pybela doesn't support running control and monitor modes at the same time. You are currently running {'CONTROL' if self._mode=='MONITOR' else 'MONITOR'} at {self.ws_ctrl_add}. You can close it running controller.disconnect()") _print_error("Connection failed") @@ -129,23 +134,33 @@ async def _async_connect(): self.ws_ctrl = await websockets.connect(self.ws_ctrl_add) self._pybela_ws_register[self._mode][self.ws_ctrl_add] = self.ws_ctrl - # Check if the response indicates a successful connection + # If connection is successful, + #  (1) send connection reply to establish the connection + # (2) connect to the data websocket + # (3) start data processing and sending tasks + # (4) start listener tasks + # (5) refresh watcher vars in case new project has been loaded in Bela response = json.loads(await self.ws_ctrl.recv()) if "event" in response and response["event"] == "connection": self.project_name = response["projectName"] + # Send connection reply to establish the connection self.send_ctrl_msg({"event": "connection-reply"}) # Connect to the data websocket self.ws_data = await websockets.connect(self.ws_data_add) - self._process_received_data_msg_worker_task = asyncio.create_task( + + # start data processing and sending tasks + self._process_received_data_msg_task = asyncio.create_task( self._process_data_msg_worker()) - self._sending_data_msg_worker_task = asyncio.create_task( + self._send_data_msg_task = asyncio.create_task( self._send_data_msg_worker()) - # Start listener loops - self._start_listener(self.ws_ctrl, self.ws_ctrl_add) - self._start_listener(self.ws_data, self.ws_data_add) + # Start listener tasks + self._ctrl_listener_task = asyncio.create_task(self._async_start_listener( + self.ws_ctrl, self.ws_ctrl_add)) + self._data_listener_task = asyncio.create_task(self._async_start_listener( + self.ws_data, self.ws_data_add)) # refresh watcher vars in case new project has been loaded in Bela self._list = self.list() @@ -162,23 +177,31 @@ async def _async_connect(): return asyncio.run(_async_connect()) - def stop(self): - """Closes websockets - """ - async def _async_stop(): - if self.ws_ctrl is not None and self.ws_ctrl.open: - await self.ws_ctrl.close() - if self.ws_data is not None and self.ws_data.open: - await self.ws_data.close() - if self._process_received_data_msg_worker_task is not None: - self._process_received_data_msg_worker_task.cancel() - if self._sending_data_msg_worker_task is not None: - self._sending_data_msg_worker_task.cancel() - return asyncio.run(_async_stop()) - def is_connected(self): return True if (self.ws_ctrl is not None and self.ws_ctrl.open) and (self.ws_data is not None and self.ws_data.open) else False + def disconnect(self): + """Closes websockets and cancels tasks + """ + + def _close_ws(): + """Closes websockets + """ + async def _async_close_ws(): + websockets = [self.ws_ctrl, self.ws_data] + for ws in websockets: + if ws is not None and ws.open: + await ws.close() + return asyncio.run(_async_close_ws()) + + _close_ws() + self._cancel_tasks( + tasks=[self._ctrl_listener_task, + self._data_listener_task, + self._process_received_data_msg_task, + self._send_data_msg_task + ]) + def list(self): """ Asks the watcher for the list of variables and their properties and returns it """ @@ -204,31 +227,31 @@ def send_ctrl_msg(self, msg): # start listener - def _start_listener(self, ws, ws_address): - """Start listener for messages. The listener is a while True loop that runs in the background and processes messages received in ws as they are received. + async def _async_start_listener(self, ws, ws_address): + """Start listener for websocket Args: - ws (websocket): Websocket object + ws (websockets.WebSocketClientProtocol): Websocket object ws_address (str): Websocket address """ - async def _async_start_listener(ws, ws_address): - try: - while ws is not None and ws.open: - msg = await ws.recv() - if self._printall_responses: - print(msg) - if ws_address == self.ws_data_add: - self._received_data_msg_queue.put_nowait(msg) - elif ws_address == self.ws_ctrl_add: - self._process_ctrl_msg(msg) - else: - print(msg) - except Exception as e: - if ws.open: # otherwise websocket was closed intentionally - _handle_connection_exception( - ws_address, e, "receiving message") - asyncio.create_task( - _async_start_listener(ws, ws_address)) + try: + while ws is not None and ws.open: + msg = await ws.recv() + if self._printall_responses: + print(msg) + if ws_address == self.ws_data_add: + self._received_data_msg_queue.put_nowait(msg) + elif ws_address == self.ws_ctrl_add: + _msg = json.loads(msg) + # response to list cmd + if "watcher" in _msg.keys() and "sampleRate" in _msg["watcher"].keys(): + self._list_response_queue.put_nowait(_msg["watcher"]) + else: + print(msg) + except Exception as e: + if ws.open: # otherwise websocket was closed intentionally + _handle_connection_exception( + ws_address, e, "receiving message") # send message @@ -263,7 +286,7 @@ async def _send_data_msg_worker(self): # process messages async def _process_data_msg_worker(self): - """Process data message. + """Process data message. Args: msg (str): Bytestring with data @@ -280,6 +303,8 @@ async def _process_data_msg(self, msg): Args: msg (str): Bytestring with data """ + _msg = json.loads(msg) + print(_msg) pass def _process_ctrl_msg(self, msg): @@ -441,7 +466,7 @@ def get_data_byte_size(self, var_type): var_type (str): Variable type Returns: - int: byte size of the variable type + int: byte size of the variable type """ data_byte_size_map = { "f": 4, @@ -460,7 +485,7 @@ def get_data_length(self, var_type, timestamp_mode): timestamp_mode (str): Timestamp mode Returns: - int: Data length in the buffer (number of elements) + int: Data length in the buffer (number of elements) """ dense_map = { "f": 1024, @@ -508,10 +533,16 @@ def get_buffer_size(self, var_type, timestamp_mode): # return error message return 0 - # destructor + def _cancel_tasks(self, tasks): + """Cancels tasks + """ + for task in tasks: + if task is not None: + task.cancel() + # destructor def __del__(self): - self.stop() # stop websockets + self.disconnect() # stop websockets def _handle_connection_exception(ws_address, exception, action): diff --git a/test/test.py b/test/test.py index b32a490..a900c94 100644 --- a/test/test.py +++ b/test/test.py @@ -8,6 +8,7 @@ # all tests should be run with Bela connected and the bela-test project (in test/bela-test) running on the board + class test_Watcher(unittest.TestCase): def setUp(self): @@ -22,7 +23,7 @@ def test_list(self): "Length of list should be equal to number of watcher variables") def test_start_stop(self): - self.watcher.stop() + self.watcher.disconnect() self.assertTrue(self.watcher.ws_ctrl.close, "Watcher ctrl websocket should be closed after stop") self.assertTrue(self.watcher.ws_data.close, @@ -121,16 +122,16 @@ def test_scheduling_streaming(self): latest_timestamp = self.streamer.get_latest_timestamp() sample_rate = self.streamer.sample_rate timestamps = [latest_timestamp + - sample_rate] * len(self.streaming_vars) # start streaming after ~1s + sample_rate] * len(self.streaming_vars) # start streaming after ~1s durations = [sample_rate] * \ len(self.streaming_vars) # stream for 1s self.streamer.schedule_streaming(variables=self.streaming_vars, - timestamps=timestamps, - durations=durations, - saving_enabled=True, - saving_dir=self.saving_dir, - saving_filename=self.saving_filename) + timestamps=timestamps, + durations=durations, + saving_enabled=True, + saving_dir=self.saving_dir, + saving_filename=self.saving_filename) self.__test_buffers(mode="schedule") @@ -177,9 +178,10 @@ def callback(block): asyncio.run(asyncio.sleep(0.5)) self.streamer.stop_streaming(variables) - - self.assertGreater(len(timestamps["myvar"]), 0, "The on_block_callback should have been called at least once") - + + self.assertGreater(len( + timestamps["myvar"]), 0, "The on_block_callback should have been called at least once") + for var in variables: for i in range(1, len(timestamps[var])): self.assertEqual(timestamps[var][i] - timestamps[var][i-1], 512, @@ -237,7 +239,7 @@ def test_logged_files_with_transfer(self): # test logged data self._test_logged_data(self.logger, self.logging_vars, - file_paths["local_paths"]) + file_paths["local_paths"]) # clean local log files for var in file_paths["local_paths"]: @@ -245,7 +247,6 @@ def test_logged_files_with_transfer(self): # clean all remote log files in project self.logger.delete_all_bin_files_in_project() - def test_logged_files_wo_transfer(self): # logging without transfer @@ -273,23 +274,21 @@ def test_logged_files_wo_transfer(self): # file_paths["remote_paths"][var]) self.logger.delete_all_bin_files_in_project() - - def test_scheduling_logging(self): latest_timestamp = self.logger.get_latest_timestamp() sample_rate = self.logger.sample_rate timestamps = [latest_timestamp + - sample_rate] * len(self.logging_vars) # start logging after ~1s + sample_rate] * len(self.logging_vars) # start logging after ~1s durations = [sample_rate] * len(self.logging_vars) # log for 1s file_paths = self.logger.schedule_logging(variables=self.logging_vars, - timestamps=timestamps, - durations=durations, - transfer=True, - logging_dir=self.logging_dir) + timestamps=timestamps, + durations=durations, + transfer=True, + logging_dir=self.logging_dir) self._test_logged_data(self.logger, self.logging_vars, - file_paths["local_paths"]) + file_paths["local_paths"]) # clean local log files for var in file_paths["local_paths"]: @@ -303,7 +302,6 @@ def test_scheduling_logging(self): # file_paths["remote_paths"][var]) - class test_Monitor(unittest.TestCase): def setUp(self): self.monitor_vars = ["myvar", "myvar2", "myvar3", "myvar4"] @@ -321,7 +319,7 @@ def test_peek(self): peeked_values = self.monitor.peek() # peeks at all variables by default for var in peeked_values: self.assertEqual(peeked_values[var]["timestamp"], peeked_values[var]["value"], - "The timestamp of the peeked variable should be equal to the value") + "The timestamp of the peeked variable should be equal to the value") def test_period_monitor(self): self.monitor.start_monitoring( @@ -337,7 +335,6 @@ def test_period_monitor(self): self.assertTrue(np.all(np.diff(monitored_values[var]["values"]) == self.period), "The values of the monitored variables should be spaced by the period") - def test_monitor_n_values(self): n_values = 25 monitored_buffer = self.monitor.monitor_n_values( @@ -354,7 +351,6 @@ def test_monitor_n_values(self): self.assertTrue(all(len(monitored_buffer[ var]["values"]) == n_values for var in self.monitor_vars[:2]), "The streaming buffers queue should have n_value for every variable") - def test_save_monitor(self): # delete any existing test files @@ -373,17 +369,16 @@ def test_save_monitor(self): for var in self.monitor_vars: loaded_buffers = self.monitor.load_data_from_file(os.path.join(self.saving_dir, - f"{var}_{self.saving_filename}")) + f"{var}_{self.saving_filename}")) self.assertEqual(loaded_buffers["timestamps"], monitored_buffers[var]["timestamps"], - "The timestamps of the loaded buffer should be equal to the timestamps of the monitored buffer") + "The timestamps of the loaded buffer should be equal to the timestamps of the monitored buffer") self.assertEqual(loaded_buffers["values"], monitored_buffers[var]["values"], - "The values of the loaded buffer should be equal to the values of the monitored buffer") + "The values of the loaded buffer should be equal to the values of the monitored buffer") for var in self.monitor_vars: remove_file(os.path.join(self.saving_dir, - f"{var}_{self.saving_filename}")) - + f"{var}_{self.saving_filename}")) class test_Controller(unittest.TestCase): @@ -400,13 +395,12 @@ def test_start_stop_controlling(self): self.controller.start_controlling(variables=self.controlled_vars) self.assertEqual(self.controller.get_controlled_status(variables=self.controlled_vars), { - var: True for var in self.controlled_vars}, "The controlled status of the variables should be True after start_controlling") + var: True for var in self.controlled_vars}, "The controlled status of the variables should be True after start_controlling") self.controller.stop_controlling(variables=self.controlled_vars) self.assertEqual(self.controller.get_controlled_status(variables=self.controlled_vars), { - var: False for var in self.controlled_vars}, "The controlled status of the variables should be False after stop_controlling") - + var: False for var in self.controlled_vars}, "The controlled status of the variables should be False after stop_controlling") def test_send_value(self): # TODO add streamer to check values are being sent @@ -430,7 +424,6 @@ def test_send_value(self): _controlled_values[var] == expected_values[idx], "The controlled value should be 4") - def remove_file(file_path): if os.path.exists(file_path): os.remove(file_path) @@ -466,7 +459,7 @@ def remove_file(file_path): test_Monitor('test_period_monitor'), test_Monitor('test_monitor_n_values'), test_Monitor('test_save_monitor'), - # controller + #  controller test_Controller('test_start_stop_controlling'), test_Controller('test_send_value') ]) From ac6b140988a4ae1ab228923fe76416e3bebe1fff Mon Sep 17 00:00:00 2001 From: Teresa Pelinski Date: Tue, 10 Dec 2024 20:24:17 +0000 Subject: [PATCH 2/5] passing all tests --- Pipfile | 2 +- Pipfile.lock | 594 ++++++++++++++++++++++++------------------- pybela/Controller.py | 18 +- pybela/Logger.py | 140 +++++----- pybela/Monitor.py | 10 +- pybela/Streamer.py | 153 ++++++----- pybela/Watcher.py | 235 +++++++++++------ requirements.txt | 2 +- test/test-send.py | 9 +- test/test.py | 31 +-- 10 files changed, 697 insertions(+), 497 deletions(-) diff --git a/Pipfile b/Pipfile index 620192a..940eedd 100644 --- a/Pipfile +++ b/Pipfile @@ -7,7 +7,7 @@ name = "pypi" jupyter = "==1.1.1" bitarray = "==3.0.0" notebook = "==7.2.2" -websockets = "==12.0" +websockets = "==14.1" ipykernel = "==6.29.5" nest-asyncio = "==1.6.0" aiofiles = "==24.1.0" diff --git a/Pipfile.lock b/Pipfile.lock index 3288f18..b18d530 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -85,10 +85,11 @@ }, "asttokens": { "hashes": [ - "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24", - "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0" + "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7", + "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2" ], - "version": "==2.4.1" + "markers": "python_version >= '3.8'", + "version": "==3.0.0" }, "async-lru": { "hashes": [ @@ -116,36 +117,34 @@ }, "bcrypt": { "hashes": [ - "sha256:096a15d26ed6ce37a14c1ac1e48119660f21b24cba457f160a4b830f3fe6b5cb", - "sha256:0da52759f7f30e83f1e30a888d9163a81353ef224d82dc58eb5bb52efcabc399", - "sha256:1bb429fedbe0249465cdd85a58e8376f31bb315e484f16e68ca4c786dcc04291", - "sha256:1d84cf6d877918620b687b8fd1bf7781d11e8a0998f576c7aa939776b512b98d", - "sha256:1ee38e858bf5d0287c39b7a1fc59eec64bbf880c7d504d3a06a96c16e14058e7", - "sha256:1ff39b78a52cf03fdf902635e4c81e544714861ba3f0efc56558979dd4f09170", - "sha256:27fe0f57bb5573104b5a6de5e4153c60814c711b29364c10a75a54bb6d7ff48d", - "sha256:3413bd60460f76097ee2e0a493ccebe4a7601918219c02f503984f0a7ee0aebe", - "sha256:3698393a1b1f1fd5714524193849d0c6d524d33523acca37cd28f02899285060", - "sha256:373db9abe198e8e2c70d12b479464e0d5092cc122b20ec504097b5f2297ed184", - "sha256:39e1d30c7233cfc54f5c3f2c825156fe044efdd3e0b9d309512cc514a263ec2a", - "sha256:3bbbfb2734f0e4f37c5136130405332640a1e46e6b23e000eeff2ba8d005da68", - "sha256:3d3a6d28cb2305b43feac298774b997e372e56c7c7afd90a12b3dc49b189151c", - "sha256:5a1e8aa9b28ae28020a3ac4b053117fb51c57a010b9f969603ed885f23841458", - "sha256:61ed14326ee023917ecd093ee6ef422a72f3aec6f07e21ea5f10622b735538a9", - "sha256:655ea221910bcac76ea08aaa76df427ef8625f92e55a8ee44fbf7753dbabb328", - "sha256:762a2c5fb35f89606a9fde5e51392dad0cd1ab7ae64149a8b935fe8d79dd5ed7", - "sha256:77800b7147c9dc905db1cba26abe31e504d8247ac73580b4aa179f98e6608f34", - "sha256:8ac68872c82f1add6a20bd489870c71b00ebacd2e9134a8aa3f98a0052ab4b0e", - "sha256:8d7bb9c42801035e61c109c345a28ed7e84426ae4865511eb82e913df18f58c2", - "sha256:8f6ede91359e5df88d1f5c1ef47428a4420136f3ce97763e31b86dd8280fbdf5", - "sha256:9c1c4ad86351339c5f320ca372dfba6cb6beb25e8efc659bedd918d921956bae", - "sha256:c02d944ca89d9b1922ceb8a46460dd17df1ba37ab66feac4870f6862a1533c00", - "sha256:c52aac18ea1f4a4f65963ea4f9530c306b56ccd0c6f8c8da0c06976e34a6e841", - "sha256:cb2a8ec2bc07d3553ccebf0746bbf3d19426d1c6d1adbd4fa48925f66af7b9e8", - "sha256:cf69eaf5185fd58f268f805b505ce31f9b9fc2d64b376642164e9244540c1221", - "sha256:f4f4acf526fcd1c34e7ce851147deedd4e26e6402369304220250598b26448db" + "sha256:041fa0155c9004eb98a232d54da05c0b41d4b8e66b6fc3cb71b4b3f6144ba837", + "sha256:04e56e3fe8308a88b77e0afd20bec516f74aecf391cdd6e374f15cbed32783d6", + "sha256:1340411a0894b7d3ef562fb233e4b6ed58add185228650942bdc885362f32c17", + "sha256:533e7f3bcf2f07caee7ad98124fab7499cb3333ba2274f7a36cf1daee7409d99", + "sha256:6765386e3ab87f569b276988742039baab087b2cdb01e809d74e74503c2faafe", + "sha256:687cf30e6681eeda39548a93ce9bfbb300e48b4d445a43db4298d2474d2a1e54", + "sha256:76132c176a6d9953cdc83c296aeaed65e1a708485fd55abf163e0d9f8f16ce0e", + "sha256:76d3e352b32f4eeb34703370e370997065d28a561e4a18afe4fef07249cb4396", + "sha256:807261df60a8b1ccd13e6599c779014a362ae4e795f5c59747f60208daddd96d", + "sha256:89df2aea2c43be1e1fa066df5f86c8ce822ab70a30e4c210968669565c0f4685", + "sha256:8ad2f4528cbf0febe80e5a3a57d7a74e6635e41af1ea5675282a33d769fba413", + "sha256:8c458cd103e6c5d1d85cf600e546a639f234964d0228909d8f8dbeebff82d526", + "sha256:8dbd0747208912b1e4ce730c6725cb56c07ac734b3629b60d4398f082ea718ad", + "sha256:909faa1027900f2252a9ca5dfebd25fc0ef1417943824783d1c8418dd7d6df4a", + "sha256:aaa2e285be097050dba798d537b6efd9b698aa88eef52ec98d23dcd6d7cf6fea", + "sha256:adadd36274510a01f33e6dc08f5824b97c9580583bd4487c564fc4617b328005", + "sha256:b1ee315739bc8387aa36ff127afc99120ee452924e0df517a8f3e4c0187a0f5f", + "sha256:b588af02b89d9fad33e5f98f7838bf590d6d692df7153647724a7f20c186f6bf", + "sha256:b7703ede632dc945ed1172d6f24e9f30f27b1b1a067f32f68bf169c5f08d0425", + "sha256:c6f5fa3775966cca251848d4d5393ab016b3afed251163c1436fefdec3b02c84", + "sha256:cde78d385d5e93ece5479a0a87f73cd6fa26b171c786a884f955e165032b262c", + "sha256:cfdf3d7530c790432046c40cda41dfee8c83e29482e6a604f8930b9930e94139", + "sha256:e158009a54c4c8bc91d5e0da80920d048f918c61a581f0a63e4e93bb556d362f", + "sha256:e84e0e6f8e40a242b11bce56c313edc2be121cec3e0ec2d76fce01f6af33c07c", + "sha256:f85b1ffa09240c89aa2e1ae9f3b1c687104f7b2b9d2098da4e923f1b7082d331" ], "markers": "python_version >= '3.7'", - "version": "==4.2.0" + "version": "==4.2.1" }, "beautifulsoup4": { "hashes": [ @@ -516,68 +515,70 @@ }, "cryptography": { "hashes": [ - "sha256:0c580952eef9bf68c4747774cde7ec1d85a6e61de97281f2dba83c7d2c806362", - "sha256:0f996e7268af62598f2fc1204afa98a3b5712313a55c4c9d434aef49cadc91d4", - "sha256:1ec0bcf7e17c0c5669d881b1cd38c4972fade441b27bda1051665faaa89bdcaa", - "sha256:281c945d0e28c92ca5e5930664c1cefd85efe80e5c0d2bc58dd63383fda29f83", - "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff", - "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805", - "sha256:443c4a81bb10daed9a8f334365fe52542771f25aedaf889fd323a853ce7377d6", - "sha256:4a02ded6cd4f0a5562a8887df8b3bd14e822a90f97ac5e544c162899bc467664", - "sha256:53a583b6637ab4c4e3591a15bc9db855b8d9dee9a669b550f311480acab6eb08", - "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e", - "sha256:74f57f24754fe349223792466a709f8e0c093205ff0dca557af51072ff47ab18", - "sha256:7e1ce50266f4f70bf41a2c6dc4358afadae90e2a1e5342d3c08883df1675374f", - "sha256:81ef806b1fef6b06dcebad789f988d3b37ccaee225695cf3e07648eee0fc6b73", - "sha256:846da004a5804145a5f441b8530b4bf35afbf7da70f82409f151695b127213d5", - "sha256:8ac43ae87929a5982f5948ceda07001ee5e83227fd69cf55b109144938d96984", - "sha256:9762ea51a8fc2a88b70cf2995e5675b38d93bf36bd67d91721c309df184f49bd", - "sha256:a2a431ee15799d6db9fe80c82b055bae5a752bef645bba795e8e52687c69efe3", - "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e", - "sha256:c2e6fc39c4ab499049df3bdf567f768a723a5e8464816e8f009f121a5a9f4405", - "sha256:cbeb489927bd7af4aa98d4b261af9a5bc025bd87f0e3547e11584be9e9427be2", - "sha256:d03b5621a135bffecad2c73e9f4deb1a0f977b9a8ffe6f8e002bf6c9d07b918c", - "sha256:d56e96520b1020449bbace2b78b603442e7e378a9b3bd68de65c782db1507995", - "sha256:df6b6c6d742395dd77a23ea3728ab62f98379eff8fb61be2744d4679ab678f73", - "sha256:e1be4655c7ef6e1bbe6b5d0403526601323420bcf414598955968c9ef3eb7d16", - "sha256:f18c716be16bc1fea8e95def49edf46b82fccaa88587a45f8dc0ff6ab5d8e0a7", - "sha256:f46304d6f0c6ab8e52770addfa2fc41e6629495548862279641972b6215451cd", - "sha256:f7b178f11ed3664fd0e995a47ed2b5ff0a12d893e41dd0494f406d1cf555cab7" - ], - "markers": "python_version >= '3.7'", - "version": "==43.0.3" + "sha256:1923cb251c04be85eec9fda837661c67c1049063305d6be5721643c22dd4e2b7", + "sha256:37d76e6863da3774cd9db5b409a9ecfd2c71c981c38788d3fcfaf177f447b731", + "sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b", + "sha256:404fdc66ee5f83a1388be54300ae978b2efd538018de18556dde92575e05defc", + "sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543", + "sha256:60eb32934076fa07e4316b7b2742fa52cbb190b42c2df2863dbc4230a0a9b385", + "sha256:62901fb618f74d7d81bf408c8719e9ec14d863086efe4185afd07c352aee1d2c", + "sha256:660cb7312a08bc38be15b696462fa7cc7cd85c3ed9c576e81f4dc4d8b2b31591", + "sha256:708ee5f1bafe76d041b53a4f95eb28cdeb8d18da17e597d46d7833ee59b97ede", + "sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb", + "sha256:831c3c4d0774e488fdc83a1923b49b9957d33287de923d58ebd3cec47a0ae43f", + "sha256:84111ad4ff3f6253820e6d3e58be2cc2a00adb29335d4cacb5ab4d4d34f2a123", + "sha256:8b3e6eae66cf54701ee7d9c83c30ac0a1e3fa17be486033000f2a73a12ab507c", + "sha256:9abcc2e083cbe8dde89124a47e5e53ec38751f0d7dfd36801008f316a127d7ba", + "sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c", + "sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285", + "sha256:abc998e0c0eee3c8a1904221d3f67dcfa76422b23620173e28c11d3e626c21bd", + "sha256:b15492a11f9e1b62ba9d73c210e2416724633167de94607ec6069ef724fad092", + "sha256:be4ce505894d15d5c5037167ffb7f0ae90b7be6f2a98f9a5c3442395501c32fa", + "sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289", + "sha256:cd4e834f340b4293430701e772ec543b0fbe6c2dea510a5286fe0acabe153a02", + "sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64", + "sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053", + "sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417", + "sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e", + "sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e", + "sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7", + "sha256:f5e7cb1e5e56ca0933b4873c0220a78b773b24d40d186b6738080b73d3d0a756", + "sha256:f677e1268c4e23420c3acade68fac427fffcb8d19d7df95ed7ad17cdef8404f4" + ], + "markers": "python_version >= '3.7' and python_full_version not in '3.9.0, 3.9.1'", + "version": "==44.0.0" }, "debugpy": { "hashes": [ - "sha256:09cc7b162586ea2171eea055985da2702b0723f6f907a423c9b2da5996ad67ba", - "sha256:0cc94186340be87b9ac5a707184ec8f36547fb66636d1029ff4f1cc020e53996", - "sha256:143ef07940aeb8e7316de48f5ed9447644da5203726fca378f3a6952a50a9eae", - "sha256:19ffbd84e757a6ca0113574d1bf5a2298b3947320a3e9d7d8dc3377f02d9f864", - "sha256:26b461123a030e82602a750fb24d7801776aa81cd78404e54ab60e8b5fecdad5", - "sha256:3a9c013077a3a0000e83d97cf9cc9328d2b0bbb31f56b0e99ea3662d29d7a6a2", - "sha256:4b93e4832fd4a759a0c465c967214ed0c8a6e8914bced63a28ddb0dd8c5f078b", - "sha256:535f4fb1c024ddca5913bb0eb17880c8f24ba28aa2c225059db145ee557035e9", - "sha256:53709d4ec586b525724819dc6af1a7703502f7e06f34ded7157f7b1f963bb854", - "sha256:5c0e5a38c7f9b481bf31277d2f74d2109292179081f11108e668195ef926c0f9", - "sha256:5c6e885dbf12015aed73770f29dec7023cb310d0dc2ba8bfbeb5c8e43f80edc9", - "sha256:64674e95916e53c2e9540a056e5f489e0ad4872645399d778f7c598eacb7b7f9", - "sha256:705cd123a773d184860ed8dae99becd879dfec361098edbefb5fc0d3683eb804", - "sha256:890fd16803f50aa9cb1a9b9b25b5ec321656dd6b78157c74283de241993d086f", - "sha256:90244598214bbe704aa47556ec591d2f9869ff9e042e301a2859c57106649add", - "sha256:a6531d952b565b7cb2fbd1ef5df3d333cf160b44f37547a4e7cf73666aca5d8d", - "sha256:b01f4a5e5c5fb1d34f4ccba99a20ed01eabc45a4684f4948b5db17a319dfb23f", - "sha256:c399023146e40ae373753a58d1be0a98bf6397fadc737b97ad612886b53df318", - "sha256:d4483836da2a533f4b1454dffc9f668096ac0433de855f0c22cdce8c9f7e10c4", - "sha256:e59b1607c51b71545cb3496876544f7186a7a27c00b436a62f285603cc68d1c6", - "sha256:e6355385db85cbd666be703a96ab7351bc9e6c61d694893206f8001e22aee091", - "sha256:ec684553aba5b4066d4de510859922419febc710df7bba04fe9e7ef3de15d34f", - "sha256:eea8821d998ebeb02f0625dd0d76839ddde8cbf8152ebbe289dd7acf2cdc6b98", - "sha256:f3cbf1833e644a3100eadb6120f25be8a532035e8245584c4f7532937edc652a", - "sha256:f95651bdcbfd3b27a408869a53fbefcc2bcae13b694daee5f1365b1b83a00113", - "sha256:ffe94dd5e9a6739a75f0b85316dc185560db3e97afa6b215628d1b6a17561cb2" + "sha256:1339e14c7d980407248f09824d1b25ff5c5616651689f1e0f0e51bdead3ea13e", + "sha256:17c5e0297678442511cf00a745c9709e928ea4ca263d764e90d233208889a19e", + "sha256:1efbb3ff61487e2c16b3e033bc8595aea578222c08aaf3c4bf0f93fadbd662ee", + "sha256:365e556a4772d7d0d151d7eb0e77ec4db03bcd95f26b67b15742b88cacff88e9", + "sha256:3d9755e77a2d680ce3d2c5394a444cf42be4a592caaf246dbfbdd100ffcf7ae5", + "sha256:3e59842d6c4569c65ceb3751075ff8d7e6a6ada209ceca6308c9bde932bcef11", + "sha256:472a3994999fe6c0756945ffa359e9e7e2d690fb55d251639d07208dbc37caea", + "sha256:54a7e6d3014c408eb37b0b06021366ee985f1539e12fe49ca2ee0d392d9ceca5", + "sha256:5e565fc54b680292b418bb809f1386f17081d1346dca9a871bf69a8ac4071afe", + "sha256:62d22dacdb0e296966d7d74a7141aaab4bec123fa43d1a35ddcb39bf9fd29d70", + "sha256:66eeae42f3137eb428ea3a86d4a55f28da9bd5a4a3d369ba95ecc3a92c1bba53", + "sha256:6953b335b804a41f16a192fa2e7851bdcfd92173cbb2f9f777bb934f49baab65", + "sha256:7c4d65d03bee875bcb211c76c1d8f10f600c305dbd734beaed4077e902606fee", + "sha256:7e646e62d4602bb8956db88b1e72fe63172148c1e25c041e03b103a25f36673c", + "sha256:7e8b079323a56f719977fde9d8115590cb5e7a1cba2fcee0986ef8817116e7c1", + "sha256:8138efff315cd09b8dcd14226a21afda4ca582284bf4215126d87342bba1cc66", + "sha256:8e99c0b1cc7bf86d83fb95d5ccdc4ad0586d4432d489d1f54e4055bcc795f693", + "sha256:957363d9a7a6612a37458d9a15e72d03a635047f946e5fceee74b50d52a9c8e2", + "sha256:957ecffff80d47cafa9b6545de9e016ae8c9547c98a538ee96ab5947115fb3dd", + "sha256:ada7fb65102a4d2c9ab62e8908e9e9f12aed9d76ef44880367bc9308ebe49a0f", + "sha256:b74a49753e21e33e7cf030883a92fa607bddc4ede1aa4145172debc637780040", + "sha256:c36856343cbaa448171cba62a721531e10e7ffb0abff838004701454149bc037", + "sha256:cc37a6c9987ad743d9c3a14fa1b1a14b7e4e6041f9dd0c8abf8895fe7a97b899", + "sha256:cfe1e6c6ad7178265f74981edf1154ffce97b69005212fbc90ca22ddfe3d017e", + "sha256:e46b420dc1bea64e5bbedd678148be512442bc589b0111bd799367cde051e71a", + "sha256:ff54ef77ad9f5c425398efb150239f6fe8e20c53ae2f68367eba7ece1e96226d" ], "markers": "python_version >= '3.8'", - "version": "==1.8.8" + "version": "==1.8.9" }, "decorator": { "hashes": [ @@ -613,10 +614,10 @@ }, "fastjsonschema": { "hashes": [ - "sha256:3d48fc5300ee96f5d116f10fe6f28d938e6008f59a6a025c2649475b87f76a23", - "sha256:5875f0b0fa7a0043a91e93a9b8f793bcbbba9691e7fd83dca95c28ba26d21f0a" + "sha256:794d4f0a58f848961ba16af7b9c85a3e88cd360df008c59aac6fc5ae9323b5d4", + "sha256:c9e5b7e908310918cf494a434eeb31384dd84a98b57a30bcb1f535015b554667" ], - "version": "==2.20.0" + "version": "==2.21.1" }, "fqdn": { "hashes": [ @@ -635,19 +636,19 @@ }, "httpcore": { "hashes": [ - "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f", - "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f" + "sha256:8551cb62a169ec7162ac7be8d4817d561f60e08eaa485234898414bb5a8a0b4c", + "sha256:a3fff8f43dc260d5bd363d9f9cf1830fa3a458b332856f34282de498ed420edd" ], "markers": "python_version >= '3.8'", - "version": "==1.0.6" + "version": "==1.0.7" }, "httpx": { "hashes": [ - "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0", - "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2" + "sha256:0858d3bab51ba7e386637f22a61d8ccddaeec5f3fe4209da3a6168dbb91573e0", + "sha256:dc0b419a0cfeb6e8b34e85167c0da2671206f5095f1baa9663d23bcfd6b535fc" ], "markers": "python_version >= '3.8'", - "version": "==0.27.2" + "version": "==0.28.0" }, "idna": { "hashes": [ @@ -714,11 +715,11 @@ }, "json5": { "hashes": [ - "sha256:1f82f36e615bc5b42f1bbd49dbc94b12563c56408c6ffa06414ea310890e9a6e", - "sha256:29c56f1accdd8bc2e037321237662034a7e07921e2b7223281a5ce2c46f0c4df" + "sha256:19b23410220a7271e8377f81ba8aacba2fdd56947fbb137ee5977cbe1f5e8dfa", + "sha256:e66941c8f0a02026943c52c2eb34ebeb2a6f819a0be05920a6f5243cd30fd559" ], "markers": "python_full_version >= '3.8.0'", - "version": "==0.9.28" + "version": "==0.10.0" }, "jsonpointer": { "hashes": [ @@ -816,11 +817,11 @@ }, "jupyterlab": { "hashes": [ - "sha256:73b6e0775d41a9fee7ee756c80f58a6bed4040869ccc21411dc559818874d321", - "sha256:ae7f3a1b8cb88b4f55009ce79fa7c06f99d70cd63601ee4aa91815d054f46f75" + "sha256:625f3ac19da91f9706baf66df25723b2f1307c1159fc7293035b066786d62a4a", + "sha256:78dd42cae5b460f377624b03966a8730e3b0692102ddf5933a2a3730c1bc0a20" ], "markers": "python_version >= '3.8'", - "version": "==4.2.5" + "version": "==4.2.6" }, "jupyterlab-pygments": { "hashes": [ @@ -939,11 +940,11 @@ }, "nbclient": { "hashes": [ - "sha256:4b3f1b7dba531e498449c4db4f53da339c91d449dc11e9af3a43b4eb5c5abb09", - "sha256:f13e3529332a1f1f81d82a53210322476a168bb7090a0289c795fe9cc11c9d3f" + "sha256:3e93e348ab27e712acd46fccd809139e356eb9a31aab641d1a7991a6eb4e6f68", + "sha256:949019b9240d66897e442888cfb618f69ef23dc71c01cb5fced8499c2cfc084d" ], "markers": "python_full_version >= '3.8.0'", - "version": "==0.10.0" + "version": "==0.10.1" }, "nbconvert": { "hashes": [ @@ -1178,11 +1179,11 @@ }, "prometheus-client": { "hashes": [ - "sha256:4fa6b4dd0ac16d58bb587c04b1caae65b8c5043e85f778f42f5f632f6af2e166", - "sha256:96c83c606b71ff2b0a433c98889d275f51ffec6c5e267de37c7a2b5c9aa9233e" + "sha256:252505a722ac04b0456be05c05f75f45d760c2911ffc45f2a06bcaed9f3ae3fb", + "sha256:594b45c410d6f4f8888940fe80b5cc2521b305a1fafe1c58609ef715a001f301" ], "markers": "python_version >= '3.8'", - "version": "==0.21.0" + "version": "==0.21.1" }, "prompt-toolkit": { "hashes": [ @@ -1506,99 +1507,112 @@ }, "rpds-py": { "hashes": [ - "sha256:031819f906bb146561af051c7cef4ba2003d28cff07efacef59da973ff7969ba", - "sha256:0626238a43152918f9e72ede9a3b6ccc9e299adc8ade0d67c5e142d564c9a83d", - "sha256:085ed25baac88953d4283e5b5bd094b155075bb40d07c29c4f073e10623f9f2e", - "sha256:0a9e0759e7be10109645a9fddaaad0619d58c9bf30a3f248a2ea57a7c417173a", - "sha256:0c025820b78817db6a76413fff6866790786c38f95ea3f3d3c93dbb73b632202", - "sha256:1ff2eba7f6c0cb523d7e9cff0903f2fe1feff8f0b2ceb6bd71c0e20a4dcee271", - "sha256:20cc1ed0bcc86d8e1a7e968cce15be45178fd16e2ff656a243145e0b439bd250", - "sha256:241e6c125568493f553c3d0fdbb38c74babf54b45cef86439d4cd97ff8feb34d", - "sha256:2c51d99c30091f72a3c5d126fad26236c3f75716b8b5e5cf8effb18889ced928", - "sha256:2d6129137f43f7fa02d41542ffff4871d4aefa724a5fe38e2c31a4e0fd343fb0", - "sha256:30b912c965b2aa76ba5168fd610087bad7fcde47f0a8367ee8f1876086ee6d1d", - "sha256:30bdc973f10d28e0337f71d202ff29345320f8bc49a31c90e6c257e1ccef4333", - "sha256:320c808df533695326610a1b6a0a6e98f033e49de55d7dc36a13c8a30cfa756e", - "sha256:32eb88c30b6a4f0605508023b7141d043a79b14acb3b969aa0b4f99b25bc7d4a", - "sha256:3b766a9f57663396e4f34f5140b3595b233a7b146e94777b97a8413a1da1be18", - "sha256:3b929c2bb6e29ab31f12a1117c39f7e6d6450419ab7464a4ea9b0b417174f044", - "sha256:3e30a69a706e8ea20444b98a49f386c17b26f860aa9245329bab0851ed100677", - "sha256:3e53861b29a13d5b70116ea4230b5f0f3547b2c222c5daa090eb7c9c82d7f664", - "sha256:40c91c6e34cf016fa8e6b59d75e3dbe354830777fcfd74c58b279dceb7975b75", - "sha256:4991ca61656e3160cdaca4851151fd3f4a92e9eba5c7a530ab030d6aee96ec89", - "sha256:4ab2c2a26d2f69cdf833174f4d9d86118edc781ad9a8fa13970b527bf8236027", - "sha256:4e8921a259f54bfbc755c5bbd60c82bb2339ae0324163f32868f63f0ebb873d9", - "sha256:4eb2de8a147ffe0626bfdc275fc6563aa7bf4b6db59cf0d44f0ccd6ca625a24e", - "sha256:5145282a7cd2ac16ea0dc46b82167754d5e103a05614b724457cffe614f25bd8", - "sha256:520ed8b99b0bf86a176271f6fe23024323862ac674b1ce5b02a72bfeff3fff44", - "sha256:52c041802a6efa625ea18027a0723676a778869481d16803481ef6cc02ea8cb3", - "sha256:5555db3e618a77034954b9dc547eae94166391a98eb867905ec8fcbce1308d95", - "sha256:58a0e345be4b18e6b8501d3b0aa540dad90caeed814c515e5206bb2ec26736fd", - "sha256:590ef88db231c9c1eece44dcfefd7515d8bf0d986d64d0caf06a81998a9e8cab", - "sha256:5afb5efde74c54724e1a01118c6e5c15e54e642c42a1ba588ab1f03544ac8c7a", - "sha256:688c93b77e468d72579351a84b95f976bd7b3e84aa6686be6497045ba84be560", - "sha256:6b4ef7725386dc0762857097f6b7266a6cdd62bfd209664da6712cb26acef035", - "sha256:6bc0e697d4d79ab1aacbf20ee5f0df80359ecf55db33ff41481cf3e24f206919", - "sha256:6dcc4949be728ede49e6244eabd04064336012b37f5c2200e8ec8eb2988b209c", - "sha256:6f54e7106f0001244a5f4cf810ba8d3f9c542e2730821b16e969d6887b664266", - "sha256:808f1ac7cf3b44f81c9475475ceb221f982ef548e44e024ad5f9e7060649540e", - "sha256:8404b3717da03cbf773a1d275d01fec84ea007754ed380f63dfc24fb76ce4592", - "sha256:878f6fea96621fda5303a2867887686d7a198d9e0f8a40be100a63f5d60c88c9", - "sha256:8a7ff941004d74d55a47f916afc38494bd1cfd4b53c482b77c03147c91ac0ac3", - "sha256:95a5bad1ac8a5c77b4e658671642e4af3707f095d2b78a1fdd08af0dfb647624", - "sha256:97ef67d9bbc3e15584c2f3c74bcf064af36336c10d2e21a2131e123ce0f924c9", - "sha256:98486337f7b4f3c324ab402e83453e25bb844f44418c066623db88e4c56b7c7b", - "sha256:98e4fe5db40db87ce1c65031463a760ec7906ab230ad2249b4572c2fc3ef1f9f", - "sha256:998a8080c4495e4f72132f3d66ff91f5997d799e86cec6ee05342f8f3cda7dca", - "sha256:9afe42102b40007f588666bc7de82451e10c6788f6f70984629db193849dced1", - "sha256:9e20da3957bdf7824afdd4b6eeb29510e83e026473e04952dca565170cd1ecc8", - "sha256:a017f813f24b9df929674d0332a374d40d7f0162b326562daae8066b502d0590", - "sha256:a429b99337062877d7875e4ff1a51fe788424d522bd64a8c0a20ef3021fdb6ed", - "sha256:a58ce66847711c4aa2ecfcfaff04cb0327f907fead8945ffc47d9407f41ff952", - "sha256:a78d8b634c9df7f8d175451cfeac3810a702ccb85f98ec95797fa98b942cea11", - "sha256:a89a8ce9e4e75aeb7fa5d8ad0f3fecdee813802592f4f46a15754dcb2fd6b061", - "sha256:a8eeec67590e94189f434c6d11c426892e396ae59e4801d17a93ac96b8c02a6c", - "sha256:aaeb25ccfb9b9014a10eaf70904ebf3f79faaa8e60e99e19eef9f478651b9b74", - "sha256:ad116dda078d0bc4886cb7840e19811562acdc7a8e296ea6ec37e70326c1b41c", - "sha256:af04ac89c738e0f0f1b913918024c3eab6e3ace989518ea838807177d38a2e94", - "sha256:af4a644bf890f56e41e74be7d34e9511e4954894d544ec6b8efe1e21a1a8da6c", - "sha256:b21747f79f360e790525e6f6438c7569ddbfb1b3197b9e65043f25c3c9b489d8", - "sha256:b229ce052ddf1a01c67d68166c19cb004fb3612424921b81c46e7ea7ccf7c3bf", - "sha256:b4de1da871b5c0fd5537b26a6fc6814c3cc05cabe0c941db6e9044ffbb12f04a", - "sha256:b80b4690bbff51a034bfde9c9f6bf9357f0a8c61f548942b80f7b66356508bf5", - "sha256:b876f2bc27ab5954e2fd88890c071bd0ed18b9c50f6ec3de3c50a5ece612f7a6", - "sha256:b8f107395f2f1d151181880b69a2869c69e87ec079c49c0016ab96860b6acbe5", - "sha256:b9b76e2afd585803c53c5b29e992ecd183f68285b62fe2668383a18e74abe7a3", - "sha256:c2b2f71c6ad6c2e4fc9ed9401080badd1469fa9889657ec3abea42a3d6b2e1ed", - "sha256:c3761f62fcfccf0864cc4665b6e7c3f0c626f0380b41b8bd1ce322103fa3ef87", - "sha256:c38dbf31c57032667dd5a2f0568ccde66e868e8f78d5a0d27dcc56d70f3fcd3b", - "sha256:ca9989d5d9b1b300bc18e1801c67b9f6d2c66b8fd9621b36072ed1df2c977f72", - "sha256:cbd7504a10b0955ea287114f003b7ad62330c9e65ba012c6223dba646f6ffd05", - "sha256:d167e4dbbdac48bd58893c7e446684ad5d425b407f9336e04ab52e8b9194e2ed", - "sha256:d2132377f9deef0c4db89e65e8bb28644ff75a18df5293e132a8d67748397b9f", - "sha256:da52d62a96e61c1c444f3998c434e8b263c384f6d68aca8274d2e08d1906325c", - "sha256:daa8efac2a1273eed2354397a51216ae1e198ecbce9036fba4e7610b308b6153", - "sha256:dc5695c321e518d9f03b7ea6abb5ea3af4567766f9852ad1560f501b17588c7b", - "sha256:de552f4a1916e520f2703ec474d2b4d3f86d41f353e7680b597512ffe7eac5d0", - "sha256:de609a6f1b682f70bb7163da745ee815d8f230d97276db049ab447767466a09d", - "sha256:e12bb09678f38b7597b8346983d2323a6482dcd59e423d9448108c1be37cac9d", - "sha256:e168afe6bf6ab7ab46c8c375606298784ecbe3ba31c0980b7dcbb9631dcba97e", - "sha256:e78868e98f34f34a88e23ee9ccaeeec460e4eaf6db16d51d7a9b883e5e785a5e", - "sha256:e860f065cc4ea6f256d6f411aba4b1251255366e48e972f8a347cf88077b24fd", - "sha256:ea3a6ac4d74820c98fcc9da4a57847ad2cc36475a8bd9683f32ab6d47a2bd682", - "sha256:ebf64e281a06c904a7636781d2e973d1f0926a5b8b480ac658dc0f556e7779f4", - "sha256:ed6378c9d66d0de903763e7706383d60c33829581f0adff47b6535f1802fa6db", - "sha256:ee1e4fc267b437bb89990b2f2abf6c25765b89b72dd4a11e21934df449e0c976", - "sha256:ee4eafd77cc98d355a0d02f263efc0d3ae3ce4a7c24740010a8b4012bbb24937", - "sha256:efec946f331349dfc4ae9d0e034c263ddde19414fe5128580f512619abed05f1", - "sha256:f414da5c51bf350e4b7960644617c130140423882305f7574b6cf65a3081cecb", - "sha256:f71009b0d5e94c0e86533c0b27ed7cacc1239cb51c178fd239c3cfefefb0400a", - "sha256:f983e4c2f603c95dde63df633eec42955508eefd8d0f0e6d236d31a044c882d7", - "sha256:faa5e8496c530f9c71f2b4e1c49758b06e5f4055e17144906245c99fa6d45356", - "sha256:fed5dfefdf384d6fe975cc026886aece4f292feaf69d0eeb716cfd3c5a4dd8be" + "sha256:009de23c9c9ee54bf11303a966edf4d9087cd43a6003672e6aa7def643d06518", + "sha256:02fbb9c288ae08bcb34fb41d516d5eeb0455ac35b5512d03181d755d80810059", + "sha256:0a0461200769ab3b9ab7e513f6013b7a97fdeee41c29b9db343f3c5a8e2b9e61", + "sha256:0b09865a9abc0ddff4e50b5ef65467cd94176bf1e0004184eb915cbc10fc05c5", + "sha256:0b8db6b5b2d4491ad5b6bdc2bc7c017eec108acbf4e6785f42a9eb0ba234f4c9", + "sha256:0c150c7a61ed4a4f4955a96626574e9baf1adf772c2fb61ef6a5027e52803543", + "sha256:0f3cec041684de9a4684b1572fe28c7267410e02450f4561700ca5a3bc6695a2", + "sha256:1352ae4f7c717ae8cba93421a63373e582d19d55d2ee2cbb184344c82d2ae55a", + "sha256:177c7c0fce2855833819c98e43c262007f42ce86651ffbb84f37883308cb0e7d", + "sha256:1978d0021e943aae58b9b0b196fb4895a25cc53d3956b8e35e0b7682eefb6d56", + "sha256:1a60bce91f81ddaac922a40bbb571a12c1070cb20ebd6d49c48e0b101d87300d", + "sha256:1aef18820ef3e4587ebe8b3bc9ba6e55892a6d7b93bac6d29d9f631a3b4befbd", + "sha256:1e9663daaf7a63ceccbbb8e3808fe90415b0757e2abddbfc2e06c857bf8c5e2b", + "sha256:20070c65396f7373f5df4005862fa162db5d25d56150bddd0b3e8214e8ef45b4", + "sha256:214b7a953d73b5e87f0ebece4a32a5bd83c60a3ecc9d4ec8f1dca968a2d91e99", + "sha256:22bebe05a9ffc70ebfa127efbc429bc26ec9e9b4ee4d15a740033efda515cf3d", + "sha256:24e8abb5878e250f2eb0d7859a8e561846f98910326d06c0d51381fed59357bd", + "sha256:26fd7cac7dd51011a245f29a2cc6489c4608b5a8ce8d75661bb4a1066c52dfbe", + "sha256:27b1d3b3915a99208fee9ab092b8184c420f2905b7d7feb4aeb5e4a9c509b8a1", + "sha256:27e98004595899949bd7a7b34e91fa7c44d7a97c40fcaf1d874168bb652ec67e", + "sha256:2b8f60e1b739a74bab7e01fcbe3dddd4657ec685caa04681df9d562ef15b625f", + "sha256:2de29005e11637e7a2361fa151f780ff8eb2543a0da1413bb951e9f14b699ef3", + "sha256:2e8b55d8517a2fda8d95cb45d62a5a8bbf9dd0ad39c5b25c8833efea07b880ca", + "sha256:2fa4331c200c2521512595253f5bb70858b90f750d39b8cbfd67465f8d1b596d", + "sha256:3445e07bf2e8ecfeef6ef67ac83de670358abf2996916039b16a218e3d95e97e", + "sha256:3453e8d41fe5f17d1f8e9c383a7473cd46a63661628ec58e07777c2fff7196dc", + "sha256:378753b4a4de2a7b34063d6f95ae81bfa7b15f2c1a04a9518e8644e81807ebea", + "sha256:3af6e48651c4e0d2d166dc1b033b7042ea3f871504b6805ba5f4fe31581d8d38", + "sha256:3dfcbc95bd7992b16f3f7ba05af8a64ca694331bd24f9157b49dadeeb287493b", + "sha256:3f21f0495edea7fdbaaa87e633a8689cd285f8f4af5c869f27bc8074638ad69c", + "sha256:4041711832360a9b75cfb11b25a6a97c8fb49c07b8bd43d0d02b45d0b499a4ff", + "sha256:44d61b4b7d0c2c9ac019c314e52d7cbda0ae31078aabd0f22e583af3e0d79723", + "sha256:4617e1915a539a0d9a9567795023de41a87106522ff83fbfaf1f6baf8e85437e", + "sha256:4b232061ca880db21fa14defe219840ad9b74b6158adb52ddf0e87bead9e8493", + "sha256:5246b14ca64a8675e0a7161f7af68fe3e910e6b90542b4bfb5439ba752191df6", + "sha256:5725dd9cc02068996d4438d397e255dcb1df776b7ceea3b9cb972bdb11260a83", + "sha256:583f6a1993ca3369e0f80ba99d796d8e6b1a3a2a442dd4e1a79e652116413091", + "sha256:59259dc58e57b10e7e18ce02c311804c10c5a793e6568f8af4dead03264584d1", + "sha256:593eba61ba0c3baae5bc9be2f5232430453fb4432048de28399ca7376de9c627", + "sha256:59f4a79c19232a5774aee369a0c296712ad0e77f24e62cad53160312b1c1eaa1", + "sha256:5f0e260eaf54380380ac3808aa4ebe2d8ca28b9087cf411649f96bad6900c728", + "sha256:62d9cfcf4948683a18a9aff0ab7e1474d407b7bab2ca03116109f8464698ab16", + "sha256:64607d4cbf1b7e3c3c8a14948b99345eda0e161b852e122c6bb71aab6d1d798c", + "sha256:655ca44a831ecb238d124e0402d98f6212ac527a0ba6c55ca26f616604e60a45", + "sha256:666ecce376999bf619756a24ce15bb14c5bfaf04bf00abc7e663ce17c3f34fe7", + "sha256:68049202f67380ff9aa52f12e92b1c30115f32e6895cd7198fa2a7961621fc5a", + "sha256:69803198097467ee7282750acb507fba35ca22cc3b85f16cf45fb01cb9097730", + "sha256:6c7b99ca52c2c1752b544e310101b98a659b720b21db00e65edca34483259967", + "sha256:6dd9412824c4ce1aca56c47b0991e65bebb7ac3f4edccfd3f156150c96a7bf25", + "sha256:70eb60b3ae9245ddea20f8a4190bd79c705a22f8028aaf8bbdebe4716c3fab24", + "sha256:70fb28128acbfd264eda9bf47015537ba3fe86e40d046eb2963d75024be4d055", + "sha256:7b2513ba235829860b13faa931f3b6846548021846ac808455301c23a101689d", + "sha256:7ef9d9da710be50ff6809fed8f1963fecdfecc8b86656cadfca3bc24289414b0", + "sha256:81e69b0a0e2537f26d73b4e43ad7bc8c8efb39621639b4434b76a3de50c6966e", + "sha256:8633e471c6207a039eff6aa116e35f69f3156b3989ea3e2d755f7bc41754a4a7", + "sha256:8bd7c8cfc0b8247c8799080fbff54e0b9619e17cdfeb0478ba7295d43f635d7c", + "sha256:9253fc214112405f0afa7db88739294295f0e08466987f1d70e29930262b4c8f", + "sha256:99b37292234e61325e7a5bb9689e55e48c3f5f603af88b1642666277a81f1fbd", + "sha256:9bd7228827ec7bb817089e2eb301d907c0d9827a9e558f22f762bb690b131652", + "sha256:9beeb01d8c190d7581a4d59522cd3d4b6887040dcfc744af99aa59fef3e041a8", + "sha256:a63cbdd98acef6570c62b92a1e43266f9e8b21e699c363c0fef13bd530799c11", + "sha256:a76e42402542b1fae59798fab64432b2d015ab9d0c8c47ba7addddbaf7952333", + "sha256:ac0a03221cdb5058ce0167ecc92a8c89e8d0decdc9e99a2ec23380793c4dcb96", + "sha256:b0b4136a252cadfa1adb705bb81524eee47d9f6aab4f2ee4fa1e9d3cd4581f64", + "sha256:b25bc607423935079e05619d7de556c91fb6adeae9d5f80868dde3468657994b", + "sha256:b3d504047aba448d70cf6fa22e06cb09f7cbd761939fdd47604f5e007675c24e", + "sha256:bb47271f60660803ad11f4c61b42242b8c1312a31c98c578f79ef9387bbde21c", + "sha256:bbb232860e3d03d544bc03ac57855cd82ddf19c7a07651a7c0fdb95e9efea8b9", + "sha256:bc27863442d388870c1809a87507727b799c8460573cfbb6dc0eeaef5a11b5ec", + "sha256:bc51abd01f08117283c5ebf64844a35144a0843ff7b2983e0648e4d3d9f10dbb", + "sha256:be2eb3f2495ba669d2a985f9b426c1797b7d48d6963899276d22f23e33d47e37", + "sha256:bf9db5488121b596dbfc6718c76092fda77b703c1f7533a226a5a9f65248f8ad", + "sha256:c58e2339def52ef6b71b8f36d13c3688ea23fa093353f3a4fee2556e62086ec9", + "sha256:cfbc454a2880389dbb9b5b398e50d439e2e58669160f27b60e5eca11f68ae17c", + "sha256:cff63a0272fcd259dcc3be1657b07c929c466b067ceb1c20060e8d10af56f5bf", + "sha256:d115bffdd417c6d806ea9069237a4ae02f513b778e3789a359bc5856e0404cc4", + "sha256:d20cfb4e099748ea39e6f7b16c91ab057989712d31761d3300d43134e26e165f", + "sha256:d48424e39c2611ee1b84ad0f44fb3b2b53d473e65de061e3f460fc0be5f1939d", + "sha256:e0fa2d4ec53dc51cf7d3bb22e0aa0143966119f42a0c3e4998293a3dd2856b09", + "sha256:e32fee8ab45d3c2db6da19a5323bc3362237c8b653c70194414b892fd06a080d", + "sha256:e35ba67d65d49080e8e5a1dd40101fccdd9798adb9b050ff670b7d74fa41c566", + "sha256:e3fb866d9932a3d7d0c82da76d816996d1667c44891bd861a0f97ba27e84fc74", + "sha256:e61b02c3f7a1e0b75e20c3978f7135fd13cb6cf551bf4a6d29b999a88830a338", + "sha256:e67ba3c290821343c192f7eae1d8fd5999ca2dc99994114643e2f2d3e6138b15", + "sha256:e79dd39f1e8c3504be0607e5fc6e86bb60fe3584bec8b782578c3b0fde8d932c", + "sha256:e89391e6d60251560f0a8f4bd32137b077a80d9b7dbe6d5cab1cd80d2746f648", + "sha256:ea7433ce7e4bfc3a85654aeb6747babe3f66eaf9a1d0c1e7a4435bbdf27fea84", + "sha256:eaf16ae9ae519a0e237a0f528fd9f0197b9bb70f40263ee57ae53c2b8d48aeb3", + "sha256:eb0c341fa71df5a4595f9501df4ac5abfb5a09580081dffbd1ddd4654e6e9123", + "sha256:f276b245347e6e36526cbd4a266a417796fc531ddf391e43574cf6466c492520", + "sha256:f47ad3d5f3258bd7058d2d506852217865afefe6153a36eb4b6928758041d831", + "sha256:f56a6b404f74ab372da986d240e2e002769a7d7102cc73eb238a4f72eec5284e", + "sha256:f5cf2a0c2bdadf3791b5c205d55a37a54025c6e18a71c71f82bb536cf9a454bf", + "sha256:f5d36399a1b96e1a5fdc91e0522544580dbebeb1f77f27b2b0ab25559e103b8b", + "sha256:f60bd8423be1d9d833f230fdbccf8f57af322d96bcad6599e5a771b151398eb2", + "sha256:f612463ac081803f243ff13cccc648578e2279295048f2a8d5eb430af2bae6e3", + "sha256:f73d3fef726b3243a811121de45193c0ca75f6407fe66f3f4e183c983573e130", + "sha256:f82a116a1d03628a8ace4859556fb39fd1424c933341a08ea3ed6de1edb0283b", + "sha256:fb0ba113b4983beac1a2eb16faffd76cb41e176bf58c4afe3e14b9c681f702de", + "sha256:fb4f868f712b2dd4bcc538b0a0c1f63a2b1d584c925e69a224d759e7070a12d5", + "sha256:fb6116dfb8d1925cbdb52595560584db42a7f664617a1f7d7f6e32f138cdf37d", + "sha256:fda7cb070f442bf80b642cd56483b5548e43d366fe3f39b98e67cce780cded00", + "sha256:feea821ee2a9273771bae61194004ee2fc33f8ec7db08117ef9147d4bbcbca8e" ], "markers": "python_version >= '3.9'", - "version": "==0.21.0" + "version": "==0.22.3" }, "send2trash": { "hashes": [ @@ -1610,19 +1624,19 @@ }, "setuptools": { "hashes": [ - "sha256:5c4ccb41111392671f02bb5f8436dfc5a9a7185e80500531b133f5775c4163ef", - "sha256:87cb777c3b96d638ca02031192d40390e0ad97737e27b6b4fa831bea86f2f829" + "sha256:8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6", + "sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d" ], "markers": "python_version >= '3.9'", - "version": "==75.5.0" + "version": "==75.6.0" }, "six": { "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.16.0" + "version": "==1.17.0" }, "sniffio": { "hashes": [ @@ -1665,36 +1679,66 @@ }, "tomli": { "hashes": [ - "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8", - "sha256:a5c57c3d1c56f5ccdf89f6523458f60ef716e210fc47c4cfb188c5ba473e0391" + "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", + "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", + "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", + "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", + "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", + "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", + "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", + "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", + "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", + "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", + "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", + "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", + "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", + "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", + "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", + "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", + "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", + "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", + "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", + "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", + "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", + "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", + "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", + "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", + "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", + "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", + "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", + "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", + "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", + "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", + "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", + "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7" ], "markers": "python_version < '3.11'", - "version": "==2.1.0" + "version": "==2.2.1" }, "tornado": { "hashes": [ - "sha256:163b0aafc8e23d8cdc3c9dfb24c5368af84a81e3364745ccb4427669bf84aec8", - "sha256:25486eb223babe3eed4b8aecbac33b37e3dd6d776bc730ca14e1bf93888b979f", - "sha256:454db8a7ecfcf2ff6042dde58404164d969b6f5d58b926da15e6b23817950fc4", - "sha256:613bf4ddf5c7a95509218b149b555621497a6cc0d46ac341b30bd9ec19eac7f3", - "sha256:6d5ce3437e18a2b66fbadb183c1d3364fb03f2be71299e7d10dbeeb69f4b2a14", - "sha256:8ae50a504a740365267b2a8d1a90c9fbc86b780a39170feca9bcc1787ff80842", - "sha256:92d3ab53183d8c50f8204a51e6f91d18a15d5ef261e84d452800d4ff6fc504e9", - "sha256:a02a08cc7a9314b006f653ce40483b9b3c12cda222d6a46d4ac63bb6c9057698", - "sha256:b24b8982ed444378d7f21d563f4180a2de31ced9d8d84443907a0a64da2072e7", - "sha256:d9a566c40b89757c9aa8e6f032bcdb8ca8795d7c1a9762910c722b1635c9de4d", - "sha256:e2e20b9113cd7293f164dc46fffb13535266e713cdb87bd2d15ddb336e96cfc4" + "sha256:072ce12ada169c5b00b7d92a99ba089447ccc993ea2143c9ede887e0937aa803", + "sha256:1a017d239bd1bb0919f72af256a970624241f070496635784d9bf0db640d3fec", + "sha256:2876cef82e6c5978fde1e0d5b1f919d756968d5b4282418f3146b79b58556482", + "sha256:304463bd0772442ff4d0f5149c6f1c2135a1fae045adf070821c6cdc76980634", + "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38", + "sha256:92bad5b4746e9879fd7bf1eb21dce4e3fc5128d71601f80005afa39237ad620b", + "sha256:932d195ca9015956fa502c6b56af9eb06106140d844a335590c1ec7f5277d10c", + "sha256:bca9eb02196e789c9cb5c3c7c0f04fb447dc2adffd95265b2c7223a8a615ccbf", + "sha256:c36e62ce8f63409301537222faffcef7dfc5284f27eec227389f2ad11b09d946", + "sha256:c82c46813ba483a385ab2a99caeaedf92585a1f90defb5693351fa7e4ea0bf73", + "sha256:e828cce1123e9e44ae2a50a9de3055497ab1d0aeb440c5ac23064d9e44880da1" ], "markers": "python_version >= '3.8'", - "version": "==6.4.1" + "version": "==6.4.2" }, "tqdm": { "hashes": [ - "sha256:0cd8af9d56911acab92182e88d763100d4788bdf421d251616040cc4d44863be", - "sha256:fe5a6f95e6fe0b9755e9469b77b9c3cf850048224ecaa8293d7d2d31f97d869a" + "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", + "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2" ], "markers": "python_version >= '3.7'", - "version": "==4.67.0" + "version": "==4.67.1" }, "traitlets": { "hashes": [ @@ -2080,7 +2124,7 @@ "sha256:4c753b3ec91717fe713c4edd522d625889d8973a349b0e582622f49766de58e6", "sha256:e67f8ac32b04be4714b42fe84ce7dad9c40985b9ca827c592cc303e7c26d9741" ], - "markers": "python_version >= '3.8'", + "markers": "platform_machine != 'ppc64le' and platform_machine != 's390x'", "version": "==25.5.0" }, "markdown-it-py": { @@ -2176,24 +2220,32 @@ }, "nh3": { "hashes": [ - "sha256:0411beb0589eacb6734f28d5497ca2ed379eafab8ad8c84b31bb5c34072b7164", - "sha256:14c5a72e9fe82aea5fe3072116ad4661af5cf8e8ff8fc5ad3450f123e4925e86", - "sha256:19aaba96e0f795bd0a6c56291495ff59364f4300d4a39b29a0abc9cb3774a84b", - "sha256:34c03fa78e328c691f982b7c03d4423bdfd7da69cd707fe572f544cf74ac23ad", - "sha256:36c95d4b70530b320b365659bb5034341316e6a9b30f0b25fa9c9eff4c27a204", - "sha256:3a157ab149e591bb638a55c8c6bcb8cdb559c8b12c13a8affaba6cedfe51713a", - "sha256:42c64511469005058cd17cc1537578eac40ae9f7200bedcfd1fc1a05f4f8c200", - "sha256:5f36b271dae35c465ef5e9090e1fdaba4a60a56f0bb0ba03e0932a66f28b9189", - "sha256:6955369e4d9f48f41e3f238a9e60f9410645db7e07435e62c6a9ea6135a4907f", - "sha256:7b7c2a3c9eb1a827d42539aa64091640bd275b81e097cd1d8d82ef91ffa2e811", - "sha256:8ce0f819d2f1933953fca255db2471ad58184a60508f03e6285e5114b6254844", - "sha256:94a166927e53972a9698af9542ace4e38b9de50c34352b962f4d9a7d4c927af4", - "sha256:a7f1b5b2c15866f2db413a3649a8fe4fd7b428ae58be2c0f6bca5eefd53ca2be", - "sha256:c8b3a1cebcba9b3669ed1a84cc65bf005728d2f0bc1ed2a6594a992e817f3a50", - "sha256:de3ceed6e661954871d6cd78b410213bdcb136f79aafe22aa7182e028b8c7307", - "sha256:f0eca9ca8628dbb4e916ae2491d72957fdd35f7a5d326b7032a345f111ac07fe" - ], - "version": "==0.2.18" + "sha256:00810cd5275f5c3f44b9eb0e521d1a841ee2f8023622de39ffc7d88bd533d8e0", + "sha256:0b6820fc64f2ff7ef3e7253a093c946a87865c877b3889149a6d21d322ed8dbd", + "sha256:11270b16c1b012677e3e2dd166c1aa273388776bf99a3e3677179db5097ee16a", + "sha256:2b926f179eb4bce72b651bfdf76f8aa05d167b2b72bc2f3657fd319f40232adc", + "sha256:2eb021804e9df1761abeb844bb86648d77aa118a663c82f50ea04110d87ed707", + "sha256:3805161c4e12088bd74752ba69630e915bc30fe666034f47217a2f16b16efc37", + "sha256:5d4f5e2189861b352b73acb803b5f4bb409c2f36275d22717e27d4e0c217ae55", + "sha256:75c7cafb840f24430b009f7368945cb5ca88b2b54bb384ebfba495f16bc9c121", + "sha256:790056b54c068ff8dceb443eaefb696b84beff58cca6c07afd754d17692a4804", + "sha256:7e98621856b0a911c21faa5eef8f8ea3e691526c2433f9afc2be713cb6fbdb48", + "sha256:833b3b5f1783ce95834a13030300cea00cbdfd64ea29260d01af9c4821da0aa9", + "sha256:a7b928862daddb29805a1010a0282f77f4b8b238a37b5f76bc6c0d16d930fd22", + "sha256:ac536a4b5c073fdadd8f5f4889adabe1cbdae55305366fb870723c96ca7f49c3", + "sha256:b8eb7affc590e542fa7981ef508cd1644f62176bcd10d4429890fc629b47f0bc", + "sha256:c2e3f0d18cc101132fe10ab7ef5c4f41411297e639e23b64b5e888ccaad63f41", + "sha256:d0adf00e2b2026fa10a42537b60d161e516f206781c7515e4e97e09f72a8c5d0", + "sha256:d53a4577b6123ca1d7e8483fad3e13cb7eda28913d516bd0a648c1a473aa21a9", + "sha256:d8325d51e47cb5b11f649d55e626d56c76041ba508cd59e0cb1cf687cc7612f1", + "sha256:df8eac98fec80bd6f5fd0ae27a65de14f1e1a65a76d8e2237eb695f9cd1121d9", + "sha256:e3dedd7858a21312f7675841529941035a2ac91057db13402c8fe907aa19205a", + "sha256:ec9c8bf86e397cb88c560361f60fdce478b5edb8b93f04ead419b72fbe937ea6", + "sha256:ed06ed78f6b69d57463b46a04f68f270605301e69d80756a8adf7519002de57d", + "sha256:fc483dd8d20f8f8c010783a25a84db3bebeadced92d24d34b40d687f8043ac69", + "sha256:fdb20740d24ab9f2a1341458a00a11205294e97e905de060eeab1ceca020c09c" + ], + "version": "==0.2.19" }, "packaging": { "hashes": [ @@ -2221,19 +2273,19 @@ }, "pipdeptree": { "hashes": [ - "sha256:6a4b4f45bb4a27a440702747636b98e4b88369c00396a840266d536fc6804b6f", - "sha256:8a9e7ceee623d1cb2839b6802c26dd40959d31ecaa1468d32616f7082658f135" + "sha256:97a455ee53cfa3dfe07223a985e4d473ac96a8b9e953d7db7f27a5e893865023", + "sha256:d520e165535e217dd8958dfc14f1922efa0f6e4ff16126a61edb7ed6c538a930" ], "index": "pypi", - "version": "==2.23.4" + "version": "==2.24.0" }, "pkginfo": { "hashes": [ - "sha256:5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297", - "sha256:889a6da2ed7ffc58ab5b900d888ddce90bce912f2d2de1dc1c26f4cb9fe65097" + "sha256:8ad91a0445a036782b9366ef8b8c2c50291f83a553478ba8580c73d3215700cf", + "sha256:dcd589c9be4da8973eceffa247733c144812759aa67eaf4bbf97016a02f39088" ], - "markers": "python_version >= '3.6'", - "version": "==1.10.0" + "markers": "python_version >= '3.8'", + "version": "==1.12.0" }, "pygments": { "hashes": [ @@ -2372,19 +2424,49 @@ }, "tomli": { "hashes": [ - "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8", - "sha256:a5c57c3d1c56f5ccdf89f6523458f60ef716e210fc47c4cfb188c5ba473e0391" + "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", + "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", + "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", + "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", + "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", + "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", + "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", + "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", + "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", + "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", + "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", + "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", + "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", + "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", + "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", + "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", + "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", + "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", + "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", + "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", + "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", + "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", + "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", + "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", + "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", + "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", + "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", + "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", + "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", + "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", + "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", + "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7" ], "markers": "python_version < '3.11'", - "version": "==2.1.0" + "version": "==2.2.1" }, "twine": { "hashes": [ - "sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997", - "sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db" + "sha256:36158b09df5406e1c9c1fb8edb24fc2be387709443e7376689b938531582ee27", + "sha256:9c6025b203b51521d53e200f4a08b116dee7500a38591668c6a6033117bdc218" ], "index": "pypi", - "version": "==5.1.1" + "version": "==6.0.1" }, "typing-extensions": { "hashes": [ diff --git a/pybela/Controller.py b/pybela/Controller.py index 582f389..25c0a98 100644 --- a/pybela/Controller.py +++ b/pybela/Controller.py @@ -33,12 +33,13 @@ def start_controlling(self, variables=[]): async def async_wait_for_control_mode_to_be_set(variables=variables): # wait for variables to be set as 'controlled' in list - _controlled_status = self.get_controlled_status( + _controlled_status = await self._async_get_controlled_status( variables) # avoid multiple calls to list while not all([_controlled_status[var] for var in variables]): - await asyncio.sleep(0.5) + await asyncio.sleep(0.2) - asyncio.run(async_wait_for_control_mode_to_be_set(variables=variables)) + self.loop.run_until_complete( + async_wait_for_control_mode_to_be_set(variables=variables)) _print_info( f"Started controlling variables {variables}... Run stop_controlling() to stop controlling the variable values.") @@ -58,12 +59,13 @@ def stop_controlling(self, variables=[]): async def async_wait_for_control_mode_to_be_set(variables=variables): # wait for variables to be set as 'uncontrolled' in list - _controlled_status = self.get_controlled_status( + _controlled_status = await self._async_get_controlled_status( variables) # avoid multiple calls to list while all([_controlled_status[var] for var in variables]): await asyncio.sleep(0.5) - asyncio.run(async_wait_for_control_mode_to_be_set(variables=variables)) + self.loop.run_until_complete( + async_wait_for_control_mode_to_be_set(variables=variables)) _print_info(f"Stopped controlling variables {variables}.") @@ -96,6 +98,12 @@ def send_value(self, variables, values): self.send_ctrl_msg( {"watcher": [{"cmd": "set", "watchers": variables, "values": values}]}) + async def _async_get_controlled_status(self, variables=[]): + """Async version of get_controller_status""" + variables = self._var_arg_checker(variables) + _list = await self._async_list() + return {var['name']: var['controlled'] for var in _list['watchers'] if var['name'] in variables} + def get_controlled_status(self, variables=[]): """Gets the controlled status (controlled or uncontrolled) of the variables diff --git a/pybela/Logger.py b/pybela/Logger.py index b4ea32c..10a487f 100644 --- a/pybela/Logger.py +++ b/pybela/Logger.py @@ -40,27 +40,23 @@ def start_logging(self, variables=[], transfer=True, logging_dir="./"): Returns: list of str: List of local paths to the logged files. """ - - remote_paths = self.__logging_common_routine( - mode="FOREVER", timestamps=[], durations=[], variables=variables, logging_dir=logging_dir) + remote_paths = self.loop.run_until_complete(self.__async_logging_common_routine( + mode="FOREVER", timestamps=[], durations=[], variables=variables, logging_dir=logging_dir)) local_paths = {} if transfer: - async def copying_tasks(): # FIXME can we remove this async? - for var in [v for v in self.watcher_vars if v["name"] in variables]: - var = var["name"] - local_path = os.path.join( - logging_dir, os.path.basename(remote_paths[var])) + for var in [v for v in self.watcher_vars if v["name"] in variables]: + var = var["name"] + local_path = os.path.join( + logging_dir, os.path.basename(remote_paths[var])) - # if file already exists, throw a warning and add number at the end of the filename - local_paths[var] = self._generate_local_filename( - local_path) + # if file already exists, throw a warning and add number at the end of the filename + local_paths[var] = self._generate_local_filename( + local_path) - copying_task = self.__copy_file_in_chunks( - remote_paths[var], local_paths[var]) - self._active_copying_tasks.append(copying_task) - - asyncio.run(copying_tasks()) + copying_task = self.__copy_file_in_chunks( + remote_paths[var], local_paths[var]) + self._active_copying_tasks.append(copying_task) return {"local_paths": local_paths, "remote_paths": remote_paths} @@ -74,18 +70,19 @@ def schedule_logging(self, variables=[], timestamps=[], durations=[], transfer=T transfer (bool, optional): Transfer files to laptop automatically during logging session. Defaults to True. logging_dir (str, optional): Path to store the files. Defaults to "./". """ - async def _async_schedule_logging(variables, timestamps, durations, transfer, logging_dir): - # checks types and if no variables are specified, stream all watcher variables (default) - latest_timestamp = self.get_latest_timestamp() - # check timestamps and duration types - assert isinstance( - timestamps, list) and all(isinstance(timestamp, int) for timestamp in timestamps), "Error: timestamps must be a list of ints." - assert isinstance( - durations, list) and all(isinstance(duration, int) for duration in durations), "Error: durations must be a list of ints." + # check timestamps and duration types + assert isinstance( + timestamps, list) and all(isinstance(timestamp, int) for timestamp in timestamps), "Error: timestamps must be a list of ints." + assert isinstance( + durations, list) and all(isinstance(duration, int) for duration in durations), "Error: durations must be a list of ints." + + remote_paths = self.loop.run_until_complete(self.__async_logging_common_routine( + mode="SCHEDULED", timestamps=timestamps, durations=durations, variables=variables, logging_dir=logging_dir)) - remote_paths = self.__logging_common_routine( - mode="SCHEDULED", timestamps=timestamps, durations=durations, variables=variables, logging_dir=logging_dir) + async def _async_schedule_logging(variables, timestamps, durations, transfer, logging_dir): + # checks types and if no variables are specified, stream all watcher variables (default) + latest_timestamp = await self._async_get_latest_timestamp() local_paths = {} if transfer: @@ -124,7 +121,7 @@ async def _async_check_if_file_exists_and_start_copying(var, timestamp): _active_checking_tasks = [] for idx, var in enumerate(variables): - check_task = asyncio.create_task( + check_task = self.loop.create_task( _async_check_if_file_exists_and_start_copying(var, timestamps[idx])) _active_checking_tasks.append(check_task) @@ -149,9 +146,9 @@ async def _async_check_if_file_exists_and_start_copying(var, timestamp): return {"local_paths": local_paths, "remote_paths": remote_paths} - return asyncio.run(_async_schedule_logging(variables=variables, timestamps=timestamps, durations=durations, transfer=transfer, logging_dir=logging_dir)) + return self.loop.run_until_complete(_async_schedule_logging(variables=variables, timestamps=timestamps, durations=durations, transfer=transfer, logging_dir=logging_dir)) - def __logging_common_routine(self, mode, timestamps=[], durations=[], variables=[], logging_dir="./"): + async def __async_logging_common_routine(self, mode, timestamps=[], durations=[], variables=[], logging_dir="./"): # checks types and if no variables are specified, stream all watcher variables (default) variables = self._var_arg_checker(variables) @@ -159,7 +156,7 @@ def __logging_common_routine(self, mode, timestamps=[], durations=[], variables= os.makedirs(logging_dir) if self.is_logging(): - self.stop_logging() + self.loop.create_task(self._async_stop_logging()) self.connect_ssh() # start ssh connection @@ -167,12 +164,12 @@ def __logging_common_routine(self, mode, timestamps=[], durations=[], variables= remote_files, remote_paths = {}, {} - self.send_ctrl_msg({"watcher": [ - {"cmd": "log", "timestamps": timestamps, "durations": durations, "watchers": variables}]}) - list_res = self.list() + await self._async_send_ctrl_msg({"watcher": [ + {"cmd": "log", "timestamps": timestamps, "durations": durations, "watchers": variables}]}) + _list = await self._async_list() for idx, var in enumerate(variables): - remote_files[var] = list_res["watchers"][idx]["logFileName"] + remote_files[var] = _list["watchers"][idx]["logFileName"] remote_paths[var] = f'/root/Bela/projects/{self.project_name}/{remote_files[var]}' _print_info( @@ -180,29 +177,30 @@ def __logging_common_routine(self, mode, timestamps=[], durations=[], variables= return remote_paths + async def _async_stop_logging(self, variables=[]): + self._logging_mode = "OFF" + if variables == []: + # if no variables specified, stop streaming all watcher variables (default) + variables = [var["name"] for var in self.watcher_vars] + + self.send_ctrl_msg( + {"watcher": [{"cmd": "unlog", "watchers": variables}]}) + + _print_info(f"Stopped logging variables {variables}...") + + await asyncio.gather(*self._active_copying_tasks, return_exceptions=True) + self._active_copying_tasks.clear() + + self.sftp_client.close() + def stop_logging(self, variables=[]): """ Stops logging session. Args: variables (list of str, optional): List of variables to stop logging. If none is passed, logging is stopped for all variables in the watcher. Defaults to []. """ - async def async_stop_logging(variables=[]): - self._logging_mode = "OFF" - if variables == []: - # if no variables specified, stop streaming all watcher variables (default) - variables = [var["name"] for var in self.watcher_vars] - - self.send_ctrl_msg( - {"watcher": [{"cmd": "unlog", "watchers": variables}]}) - - _print_info(f"Stopped logging variables {variables}...") - - await asyncio.gather(*self._active_copying_tasks, return_exceptions=True) - self._active_copying_tasks.clear() - - self.sftp_client.close() - return asyncio.run(async_stop_logging(variables)) + return self.loop.run_until_complete(self._async_stop_logging(variables)) def connect_ssh(self): """ Connects to Bela via ssh to transfer log files. @@ -371,7 +369,7 @@ async def async_copy_file_in_chunks(remote_path, local_path, chunk_size=2**12): finally: await self._async_remove_item_from_list(self._active_copying_tasks, asyncio.current_task()) - return asyncio.create_task(async_copy_file_in_chunks(remote_path, local_path, chunk_size)) + return self.loop.create_task(async_copy_file_in_chunks(remote_path, local_path, chunk_size)) def copy_file_from_bela(self, remote_path, local_path, verbose=True): """Copy a file from Bela onto the local machine. @@ -382,7 +380,7 @@ def copy_file_from_bela(self, remote_path, local_path, verbose=True): verbose (bool, optional): Show info messages. Defaults to True. """ self.connect_ssh() - asyncio.run(self._async_copy_file_from_bela( + self.loop.run_until_complete(self._async_copy_file_from_bela( remote_path, local_path, verbose)) self.disconnect_ssh() @@ -400,7 +398,7 @@ def copy_all_bin_files_in_project(self, dir="./", verbose=True): "copy", dir) # wait until all files are copied - asyncio.run(asyncio.gather( + self.loop.run_until_complete(asyncio.gather( *copy_tasks, return_exceptions=True)) if verbose: @@ -422,7 +420,7 @@ async def _async_copy_file_from_bela(self, remote_path, local_path, verbose=Fals try: if os.path.exists(local_path): local_path = self._generate_local_filename(local_path) - transferred_event = asyncio.Event() + transferred_event = asyncio.Event(loop=self.loop) def callback(transferred, to_transfer): return transferred_event.set( ) if transferred == to_transfer else None self.sftp_client.get(remote_path, local_path, callback=callback) @@ -494,7 +492,8 @@ def delete_file_from_bela(self, remote_path, verbose=True): remote_path (str): Path to the remote file to be deleted. """ self.connect_ssh() - asyncio.run(self._async_delete_file_from_bela(remote_path, verbose)) + self.loop.run_until_complete( + self._async_delete_file_from_bela(remote_path, verbose)) self.disconnect_ssh() def delete_all_bin_files_in_project(self, verbose=True): @@ -507,7 +506,7 @@ def delete_all_bin_files_in_project(self, verbose=True): "delete") # wait until all files are deleted - asyncio.run(asyncio.gather( + self.loop.run_until_complete(asyncio.gather( *deletion_tasks, return_exceptions=True)) if verbose: @@ -557,22 +556,19 @@ def _action_on_all_bin_files_in_project(self, action, local_dir=None): # Iterate through the files and delete .bin files tasks = [] - async def _async_action_action_on_all_bin_files_in_project(): # FIXME can we avoid this async? - for file_name in file_list: - if file_name.endswith('.bin'): - remote_file_path = f"{remote_path}/{file_name}" - if action == "delete": - task = asyncio.create_task( - self._async_delete_file_from_bela(remote_file_path)) - elif action == "copy": - local_filename = os.path.join(local_dir, file_name) - task = asyncio.create_task( - self._async_copy_file_from_bela(remote_file_path, local_filename)) - else: - raise ValueError(f"Invalid action: {action}") - tasks.append(task) - - asyncio.run(_async_action_action_on_all_bin_files_in_project()) + for file_name in file_list: + if file_name.endswith('.bin'): + remote_file_path = f"{remote_path}/{file_name}" + if action == "delete": + task = self.loop.create_task( + self._async_delete_file_from_bela(remote_file_path)) + elif action == "copy": + local_filename = os.path.join(local_dir, file_name) + task = self.loop.create_task( + self._async_copy_file_from_bela(remote_file_path, local_filename)) + else: + raise ValueError(f"Invalid action: {action}") + tasks.append(task) return tasks diff --git a/pybela/Monitor.py b/pybela/Monitor.py index 26bdad8..846910a 100644 --- a/pybela/Monitor.py +++ b/pybela/Monitor.py @@ -1,5 +1,4 @@ from .Streamer import Streamer -import asyncio class Monitor(Streamer): @@ -57,11 +56,11 @@ async def _async_peek(variables): peeked_values = self._peek_response # set _peek_response again to None so that peek is not notified every time a new buffer is received (see Streamer._process_data_msg) self._peek_response = None - self.stop_monitoring(variables) + await self._async_stop_monitoring(variables) return peeked_values - return asyncio.run(_async_peek(variables)) + return self.loop.run_until_complete(_async_peek(variables)) # using list # res = self.list() @@ -139,6 +138,11 @@ async def async_monitor_n_values(self, variables=[], periods=[], n_values=1000, self.async_stream_n_values(variables, periods, n_values, saving_enabled, saving_filename) + async def _async_stop_monitoring(self, variables=[]): + await self._async_stop_streaming(variables) + self._monitored_vars = None + return {var: self.values[var] for var in self.values if self.values[var]["timestamps"] != []} + def stop_monitoring(self, variables=[]): """ Stops the current monitoring session for the given variables. If no variables are passed, the monitoring of all variables is interrupted. diff --git a/pybela/Streamer.py b/pybela/Streamer.py index 75f2c84..b481427 100644 --- a/pybela/Streamer.py +++ b/pybela/Streamer.py @@ -34,14 +34,14 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add # -- streaming -- self._streaming_mode = "OFF" # OFF, FOREVER, N_VALUES, PEEK :: this flag prevents writing into the streaming buffer unless requested by the user using the start/stop_streaming() functions - self._streaming_buffer_available = asyncio.Event() + self._streaming_buffer_available = asyncio.Event(loop=self.loop) # number of streaming buffers (not of data points!) self._streaming_buffers_queue_length = 1000 self._streaming_buffers_queue = None self.last_streamed_buffer = {} # -- on data/block callbacks -- - self._processed_data_msg_queue = asyncio.Queue() + self._processed_data_msg_queue = asyncio.Queue(loop=self.loop) self._on_buffer_callback_is_active = False self._on_buffer_callback_worker_task = None self._on_block_callback_is_active = False @@ -57,7 +57,7 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add # -- monitor -- # stores the list of monitored variables for each monitored session. cleaned after each monitoring session. used to avoid calling list() every time a new message is parsed self._monitored_vars = None - self._peek_response_available = asyncio.Event() + self._peek_response_available = asyncio.Event(loop=self.loop) self._peek_response = None self._mode = "STREAM" @@ -81,6 +81,15 @@ def monitored_vars(self): self._monitored_vars = None return self._monitored_vars + async def _async_monitored_vars(self): + if self._monitored_vars is None: + _list = await self._async_list() + self._monitored_vars = self._filtered_watcher_vars( + _list["watchers"], lambda var: var["monitor"]) + if self._monitored_vars == []: + self._monitored_vars = None + return self._monitored_vars + @property def streaming_buffers_queue_length(self): """ @@ -146,7 +155,8 @@ def __streaming_common_routine(self, variables=[], saving_enabled=False, saving_ self._saving_filename = self._generate_filename( saving_filename, saving_dir) if saving_enabled else None - self._processed_data_msg_queue = asyncio.Queue() # clear processed data queue + self._processed_data_msg_queue = asyncio.Queue( + loop=self.loop) # clear processed data queue async def async_callback_workers(): @@ -156,15 +166,15 @@ async def async_callback_workers(): return 0 if on_buffer_callback: self._on_buffer_callback_is_active = True - self._on_buffer_callback_worker_task = asyncio.create_task( + self._on_buffer_callback_worker_task = self.loop.create_task( self.__async_on_buffer_callback_worker(on_buffer_callback, callback_args)) elif on_block_callback: self._on_block_callback_is_active = True - self._on_block_callback_worker_task = asyncio.create_task( + self._on_block_callback_worker_task = self.loop.create_task( self.__async_on_block_callback_worker(on_block_callback, callback_args, variables)) - asyncio.run(async_callback_workers()) + self.loop.create_task(async_callback_workers()) # checks types and if no variables are specified, stream all watcher variables (default) return self._var_arg_checker(variables) @@ -204,9 +214,11 @@ def start_streaming(self, variables=[], periods=[], saving_enabled=False, saving if periods != []: warnings.warn( "Periods list is ignored in streaming mode STREAM") - self.send_ctrl_msg( + self._to_send_ctrl_msg_queue.put_nowait( {"watcher": [{"cmd": "watch", "watchers": variables}]}) - # asyncio.run(async_wait_for_streaming_to_start()) + # self.send_ctrl_msg( + # {"watcher": [{"cmd": "watch", "watchers": variables}]}) + # # asyncio.run(async_wait_for_streaming_to_start()) _print_info( f"Started streaming variables {variables}... Run stop_streaming() to stop streaming.") elif self._mode == "MONITOR": @@ -220,6 +232,40 @@ def start_streaming(self, variables=[], periods=[], saving_enabled=False, saving elif self._streaming_mode == "PEEK": _print_info(f"Peeking at variables {variables}...") + async def _async_stop_streaming(self, variables=[]): + _previous_streaming_mode = copy.copy(self._streaming_mode) + + self._streaming_mode = "OFF" + + if self._saving_enabled: + self._saving_enabled = False + self._saving_filename = None + # await all active saving tasks + await asyncio.gather(*self._active_saving_tasks, return_exceptions=True, loop=self.loop) + self._active_saving_tasks.clear() + + if variables == []: + # if no variables specified, stop streaming all watcher variables (default) + variables = [var["name"] for var in self.watcher_vars] + + if self._mode == "STREAM" and _previous_streaming_mode != "SCHEDULE": + self.send_ctrl_msg( + {"watcher": [{"cmd": "unwatch", "watchers": variables}]}) + _print_info(f"Stopped streaming variables {variables}...") + elif self._mode == "MONITOR" and _previous_streaming_mode != "SCHEDULE": + self.send_ctrl_msg( + {"watcher": [{"cmd": "monitor", "periods": [0]*len(variables), "watchers": variables}]}) # setting period to 0 disables monitoring + if not _previous_streaming_mode == "PEEK": + _print_info(f"Stopped monitoring variables {variables}...") + self._processed_data_msg_queue = asyncio.Queue( + loop=self.loop) # clear processed data queue + self._on_buffer_callback_is_active = False + if self._on_buffer_callback_worker_task: + await self._on_buffer_callback_worker_task.cancel() + self._on_block_callback_is_active = False + if self._on_block_callback_worker_task: + await self._on_block_callback_worker_task.cancel() + def stop_streaming(self, variables=[]): """ Stops the current streaming session for the given variables. If no variables are passed, the streaming of all variables is interrupted. @@ -230,40 +276,8 @@ def stop_streaming(self, variables=[]): Returns: streaming_buffers_queue (dict): Dict containing the streaming buffers for each streamed variable. """ - async def async_stop_streaming(variables=[]): - _previous_streaming_mode = copy.copy(self._streaming_mode) - - self._streaming_mode = "OFF" - - if self._saving_enabled: - self._saving_enabled = False - self._saving_filename = None - # await all active saving tasks - await asyncio.gather(*self._active_saving_tasks, return_exceptions=True) - self._active_saving_tasks.clear() - - if variables == []: - # if no variables specified, stop streaming all watcher variables (default) - variables = [var["name"] for var in self.watcher_vars] - - if self._mode == "STREAM" and _previous_streaming_mode != "SCHEDULE": - self.send_ctrl_msg( - {"watcher": [{"cmd": "unwatch", "watchers": variables}]}) - _print_info(f"Stopped streaming variables {variables}...") - elif self._mode == "MONITOR" and _previous_streaming_mode != "SCHEDULE": - self.send_ctrl_msg( - {"watcher": [{"cmd": "monitor", "periods": [0]*len(variables), "watchers": variables}]}) # setting period to 0 disables monitoring - if not _previous_streaming_mode == "PEEK": - _print_info(f"Stopped monitoring variables {variables}...") - self._processed_data_msg_queue = asyncio.Queue() # clear processed data queue - self._on_buffer_callback_is_active = False - if self._on_buffer_callback_worker_task: - self._on_buffer_callback_worker_task.cancel() - self._on_block_callback_is_active = False - if self._on_block_callback_worker_task: - self._on_block_callback_worker_task.cancel() - - return asyncio.run(async_stop_streaming(variables)) + + return self.loop.run_until_complete(self._async_stop_streaming(variables)) def schedule_streaming(self, variables=[], timestamps=[], durations=[], saving_enabled=False, saving_filename="var_stream.txt", saving_dir="./", on_buffer_callback=None, on_block_callback=None, callback_args=()): """Schedule streaming of variables. The streaming session can be stopped with stop_streaming(). @@ -295,22 +309,22 @@ async def async_check_if_variables_have_been_streamed_and_stop(): finished_streaming_vars = [] while not all(var in finished_streaming_vars for var in variables): - - for var in [v["name"] for v in self.watched_vars]: + _watched_vars = await self._async_watched_vars() + for var in [v["name"] for v in _watched_vars]: if var not in started_streaming_vars: started_streaming_vars.append(var) _print_info(f"Started streaming {var}...") for var in started_streaming_vars: - if var not in [v["name"] for v in self.watched_vars]: + if var not in [v["name"] for v in _watched_vars]: finished_streaming_vars.append(var) _print_info(f"Stopped streaming {var}") await asyncio.sleep(0.1) - self.stop_streaming() + await self._async_stop_streaming() - asyncio.run( + self.loop.run_until_complete( async_check_if_variables_have_been_streamed_and_stop()) def stream_n_values(self, variables=[], periods=[], n_values=1000, saving_enabled=False, saving_filename=None, saving_dir="./", on_buffer_callback=None, on_block_callback=None, callback_args=()): @@ -320,7 +334,7 @@ def stream_n_values(self, variables=[], periods=[], n_values=1000, saving_enable Note: This function will block the main thread until n_values have been streamed. Since the streamed values come in blocks, the actual number of returned frames streamed may be higher than n_values, unless n_values is a multiple of the block size (streamer._streaming_block_size). To avoid blocking, use the async version of this function: - stream_task = asyncio.create_task(streamer.async_stream_n_values(variables, n_values, periods, saving_enabled, saving_filename)) + stream_task = self.loop.create_task(streamer.async_stream_n_values(variables, n_values, periods, saving_enabled, saving_filename)) and retrieve the streaming buffer using: streaming_buffers_queue = await stream_task @@ -339,12 +353,12 @@ def stream_n_values(self, variables=[], periods=[], n_values=1000, saving_enable Returns: streaming_buffers_queue (dict): Dict containing the streaming buffers for each streamed variable. """ - return asyncio.run(self.async_stream_n_values(variables, periods, n_values, saving_enabled, saving_filename, saving_dir, on_buffer_callback, on_block_callback, callback_args)) + return self.loop.run_until_complete(self.async_stream_n_values(variables, periods, n_values, saving_enabled, saving_filename, saving_dir, on_buffer_callback, on_block_callback, callback_args)) async def async_stream_n_values(self, variables=[], periods=[], n_values=1000, saving_enabled=False, saving_filename="var_stream.txt", saving_dir="./", on_buffer_callback=None, on_block_callback=None, callback_args=()): """ Asynchronous version of stream_n_values(). Usage: - stream_task = asyncio.create_task(streamer.async_stream_n_values(variables, n_values, saving_enabled, saving_filename)) + stream_task = self.loop.create_task(streamer.async_stream_n_values(variables, n_values, saving_enabled, saving_filename)) and retrieve the streaming buffer using: streaming_buffers_queue = await stream_task @@ -388,7 +402,7 @@ async def async_stream_n_values(self, variables=[], periods=[], n_values=1000, s if periods != []: warnings.warn( "Periods list is ignored in streaming mode STREAM") - self.send_ctrl_msg( + self._to_send_ctrl_msg_queue.put_nowait( {"watcher": [{"cmd": "unwatch", "watchers": [var["name"] for var in self.watcher_vars]}, {"cmd": "watch", "watchers": variables}]}) _print_info( f"Streaming {n_values} values for variables {variables}...") @@ -398,7 +412,7 @@ async def async_stream_n_values(self, variables=[], periods=[], n_values=1000, s self.streaming_buffers_queue_length = n_values periods = self._check_periods(periods, variables) - self.send_ctrl_msg( + self._to_send_ctrl_msg_queue.put_nowait( {"watcher": [{"cmd": "monitor", "watchers": variables, "periods": periods}]}) _print_info( f"Monitoring {n_values} values for variables {variables} with periods {periods}...") @@ -408,7 +422,7 @@ async def async_stream_n_values(self, variables=[], periods=[], n_values=1000, s self._streaming_buffer_available.clear() # turns off listener, unwatches variables - self.stop_streaming(variables) + await self._async_stop_streaming(variables) if self._mode == "MONITOR": self._monitored_vars = None # reset monitored vars @@ -447,7 +461,7 @@ async def __async_on_block_callback_worker(self, on_block_callback, callback_arg msgs = [] for var in variables: # if not self._processed_data_msg_queue.empty(): - msg = await asyncio.wait_for(self._processed_data_msg_queue.get(), timeout=1) + msg = await self._processed_data_msg_queue.get() msgs.append(msg) self._processed_data_msg_queue.task_done() if len(msgs) == len(variables): @@ -619,7 +633,7 @@ def plot_data(self, x_var, y_vars, y_range=None, plot_update_delay=100, rollover # check that x_var and y_vars are either streamed or monitored for _var in [x_var, *y_vars]: - if not (_var in [var["name"] for var in self.watched_vars] or _var in [var["name"] for var in self.monitored_vars]): + if not (_var in [var["name"] for var in self.watched_vars] or _var in [var["name"] for var in self.monitored_vars]): # FIXME _print_error( f"PlottingError: {_var} is not being streamed or monitored.") return @@ -711,7 +725,7 @@ async def _process_data_msg(self, msg): _saving_var_filename = os.path.join(os.path.dirname( self._saving_filename), f"{var_name}_{os.path.basename(self._saving_filename)}") # save the data asynchronously - saving_task = asyncio.create_task( + saving_task = self.loop.create_task( self._save_data_to_file(_saving_var_filename, parsed_buffer)) self._active_saving_tasks.append(saving_task) @@ -726,7 +740,9 @@ async def _process_data_msg(self, msg): # if streaming buffers queue is full for watched variables and streaming mode is n_values if self._streaming_mode == "N_VALUES": - obs_vars = self.watched_vars if self._mode == "STREAM" else self.monitored_vars + _watched_vars = await self._async_watched_vars() + _monitored_vars = await self._async_monitored_vars() + obs_vars = _watched_vars if self._mode == "STREAM" else _monitored_vars if all(len(self._streaming_buffers_queue[var["name"]]) == self._streaming_buffers_queue_length for var in obs_vars): self._streaming_mode = "OFF" @@ -754,8 +770,8 @@ async def _save_data_to_file(self, filename, msg): except Exception as e: _print_error(f"Error while saving data to file: {e}") - finally: - await self._async_remove_item_from_list(self._active_saving_tasks, asyncio.current_task()) + # finally: + # await self._async_remove_item_from_list(self._active_saving_tasks, asyncio.current_task()) def _generate_filename(self, saving_filename, saving_dir="./"): """ Generates a filename for saving data by adding the variable name and a number at the end in case the filename already exists to avoid overwriting saved data. Pattern: varname_filename__idx.ext. This function is called by start_streaming() and stream_n_values() when saving is enabled. @@ -819,8 +835,15 @@ def _check_periods(self, periods, variables): return periods - def __del__(self): - super().__del__() - self._cancel_tasks(tasks=[ - self._on_buffer_callback_worker_task, - self._on_block_callback_worker_task]) + async def _async_disconnect(self): + await super()._async_disconnect() + if self._on_buffer_callback_worker_task is not None and not self._on_buffer_callback_worker_task.done(): + self._on_buffer_callback_worker_task.cancel() + if self._on_block_callback_worker_task is not None and not self._on_block_callback_worker_task.done(): + self._on_block_callback_worker_task.cancel() + + def disconnect(self): + self.loop.run_until_complete(self._async_disconnect()) + + # def __del__(self): + # super().__del__() diff --git a/pybela/Watcher.py b/pybela/Watcher.py index dc88743..5cd5d8d 100644 --- a/pybela/Watcher.py +++ b/pybela/Watcher.py @@ -5,6 +5,7 @@ import errno import struct import os +import gc from .utils import _print_error, _print_warning, _print_ok @@ -31,39 +32,65 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add self.ws_data_add = f"ws://{self.ip}:{self.port}/{self.data_add}" self.ws_ctrl = None self.ws_data = None + global _pybela_ws_register + try: + _ = _pybela_ws_register + except NameError: # initialise _pybela_ws_register only once in runtime + _pybela_ws_register = {"loop": None, + "WATCH": {}, + "STREAM": {}, + "LOG": {}, + "MONITOR": {}, + "CONTROL": {}} + + self._pybela_ws_register = _pybela_ws_register + + # background event loop + + # If no loop exists, create a new one + if self._pybela_ws_register["loop"] is None: + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + self._pybela_ws_register["loop"] = self.loop + else: + self.loop = self._pybela_ws_register["loop"] + # self.loop.run_forever() + # self.loop = asyncio.new_event_loop() + # asyncio.set_event_loop(self.loop) + # self.loop.run_forever() + + # self.loop_thread = threading.Thread( + # target=self._run_event_loop, daemon=True) + # self.loop_thread.start() + # self.loop = asyncio.get_event_loop() # tasks self._ctrl_listener_task = None self._data_listener_task = None self._process_received_data_msg_task = None self._send_data_msg_task = None + self._send_ctrl_msg_task = None # queues - self._received_data_msg_queue = asyncio.Queue() - self._list_response_queue = asyncio.Queue() - self._to_send_data_msg_queue = asyncio.Queue() + self._received_data_msg_queue = asyncio.Queue(loop=self.loop) + self._list_response_queue = asyncio.Queue(loop=self.loop) + self._to_send_data_msg_queue = asyncio.Queue(loop=self.loop) + self._to_send_ctrl_msg_queue = asyncio.Queue(loop=self.loop) self._watcher_vars = None self._mode = "WATCH" - global _pybela_ws_register - try: - _ = _pybela_ws_register - except NameError: # initialise _pybela_ws_register only once in runtime - _pybela_ws_register = {"WATCH": {}, - "STREAM": {}, - "LOG": {}, - "MONITOR": {}, - "CONTROL": {}} - - self._pybela_ws_register = _pybela_ws_register - # debug self._printall_responses = False # event loop needs to be nested - otherwise it conflicts with jupyter's event loop - nest_asyncio.apply() + # nest_asyncio.apply() + def _run_event_loop(self): + asyncio.set_event_loop(self.loop) + self.loop.run_forever() + + # properties @property def sample_rate(self): @@ -82,6 +109,13 @@ def watcher_vars(self): _list["watchers"], lambda var: True) return self._watcher_vars # updates every time start is called + async def _async_watcher_vars(self): + if self._watcher_vars == None: + _list = await self._async_list() + self._watcher_vars = self._filtered_watcher_vars( + _list["watchers"], lambda var: True) + return self._watcher_vars + @property def watched_vars(self): """Returns a list of the variables in the watcher that are being watched (i.e., whose data is being sent over websockets for streaming or monitoring) @@ -92,6 +126,10 @@ def watched_vars(self): _list = self.list() return self._filtered_watcher_vars(_list["watchers"], lambda var: var["watched"]) + async def _async_watched_vars(self): + _list = await self._async_list() + return self._filtered_watcher_vars(_list["watchers"], lambda var: var["watched"]) + @property def unwatched_vars(self): """Returns a list of the variables in the watcher that are not being watched (i.e., whose data is NOT being sent over websockets for streaming or monitoring) @@ -102,6 +140,10 @@ def unwatched_vars(self): _list = self.list() return self._filtered_watcher_vars(_list["watchers"], lambda var: not var["watched"]) + async def _async_unwatched_vars(self): + _list = await self._async_list() + return self._filtered_watcher_vars(_list["watchers"], lambda var: not var["watched"]) + # --- public methods --- # def connect(self): @@ -114,16 +156,18 @@ def connect(self): async def _async_connect(): try: # Close any open ctrl websocket open for the same mode (STREAM, LOG, MONITOR, WATCH) - if self._pybela_ws_register[self._mode].get(self.ws_ctrl_add) is not None and self._pybela_ws_register[self._mode][self.ws_ctrl_add].open: + if self._pybela_ws_register[self._mode].get(self.ws_ctrl_add) is not None and self._pybela_ws_register[self._mode][self.ws_ctrl_add].state == 1: _print_warning( f"pybela doesn't support more than one active connection at a time for a given mode. Closing previous connection for {self._mode} at {self.ws_ctrl_add}.") await self._pybela_ws_register[self._mode][self.ws_ctrl_add].close() + self._pybela_ws_register[self._mode][self.ws_ctrl_add].keepalive_task.cancel( + ) # Control and monitor can't be used at the same time _is_control_mode_running = self._pybela_ws_register["CONTROL"].get( - self.ws_ctrl_add) is not None and self._pybela_ws_register["CONTROL"][self.ws_ctrl_add].open + self.ws_ctrl_add) is not None and self._pybela_ws_register["CONTROL"][self.ws_ctrl_add].state == 1 _is_monitor_mode_running = self._pybela_ws_register["MONITOR"].get( - self.ws_ctrl_add) is not None and self._pybela_ws_register["MONITOR"][self.ws_ctrl_add].open + self.ws_ctrl_add) is not None and self._pybela_ws_register["MONITOR"][self.ws_ctrl_add].state == 1 if (self._mode == "MONITOR" and _is_control_mode_running) or (self._mode == "CONTROL" and _is_monitor_mode_running): _print_warning( f"pybela doesn't support running control and monitor modes at the same time. You are currently running {'CONTROL' if self._mode=='MONITOR' else 'MONITOR'} at {self.ws_ctrl_add}. You can close it running controller.disconnect()") @@ -131,7 +175,11 @@ async def _async_connect(): return 0 # Connect to the control websocket + # try: self.ws_ctrl = await websockets.connect(self.ws_ctrl_add) + # except asyncio.TimeoutError: + # _print_error(f"Timeout connecting to {self.ws_ctrl_add}") + # return 0 self._pybela_ws_register[self._mode][self.ws_ctrl_add] = self.ws_ctrl # If connection is successful, @@ -145,25 +193,27 @@ async def _async_connect(): self.project_name = response["projectName"] # Send connection reply to establish the connection - self.send_ctrl_msg({"event": "connection-reply"}) + await self._async_send_ctrl_msg({"event": "connection-reply"}) # Connect to the data websocket self.ws_data = await websockets.connect(self.ws_data_add) # start data processing and sending tasks - self._process_received_data_msg_task = asyncio.create_task( + self._process_received_data_msg_task = self.loop.create_task( self._process_data_msg_worker()) - self._send_data_msg_task = asyncio.create_task( + self._send_data_msg_task = self.loop.create_task( self._send_data_msg_worker()) + self._send_ctrl_msg_task = self.loop.create_task( + self._send_ctrl_msg_worker()) # Start listener tasks - self._ctrl_listener_task = asyncio.create_task(self._async_start_listener( + self._ctrl_listener_task = self.loop.create_task(self._async_start_listener( self.ws_ctrl, self.ws_ctrl_add)) - self._data_listener_task = asyncio.create_task(self._async_start_listener( + self._data_listener_task = self.loop.create_task(self._async_start_listener( self.ws_data, self.ws_data_add)) # refresh watcher vars in case new project has been loaded in Bela - self._list = self.list() + self._list = await self._async_list() self._sample_rate = self._list["sampleRate"] self._watcher_vars = self._filtered_watcher_vars(self._list["watchers"], lambda var: True) @@ -175,45 +225,40 @@ async def _async_connect(): except Exception as e: raise ConnectionError(f"Connection failed: {str(e)}.") - return asyncio.run(_async_connect()) + return self.loop.run_until_complete(_async_connect()) def is_connected(self): - return True if (self.ws_ctrl is not None and self.ws_ctrl.open) and (self.ws_data is not None and self.ws_data.open) else False + return True if (self.ws_ctrl is not None and self.ws_ctrl.state == 1) and (self.ws_data is not None and self.ws_data.state == 1) else False + + async def _async_disconnect(self): + # close websockets + wss = [self.ws_ctrl, self.ws_data] + for ws in wss: + if ws is not None and ws.state == 1: + await ws.close() + ws.keepalive_task.cancel() # cancel keepalive task def disconnect(self): - """Closes websockets and cancels tasks + """Closes websockets """ + self.loop.run_until_complete(self._async_disconnect()) - def _close_ws(): - """Closes websockets - """ - async def _async_close_ws(): - websockets = [self.ws_ctrl, self.ws_data] - for ws in websockets: - if ws is not None and ws.open: - await ws.close() - return asyncio.run(_async_close_ws()) - - _close_ws() - self._cancel_tasks( - tasks=[self._ctrl_listener_task, - self._data_listener_task, - self._process_received_data_msg_task, - self._send_data_msg_task - ]) + pass + + async def _async_list(self): + await self._async_send_ctrl_msg({"watcher": [{"cmd": "list"}]}) + # Wait for the list response to be available + list_res = await self._list_response_queue.get() + self._list_response_queue.task_done() + return list_res def list(self): """ Asks the watcher for the list of variables and their properties and returns it """ - async def _async_list(): - self.send_ctrl_msg({"watcher": [{"cmd": "list"}]}) - # Wait for the list response to be available - - list_res = await self._list_response_queue.get() - self._list_response_queue.task_done() - return list_res + return self.loop.run_until_complete(self._async_list()) - return asyncio.run(_async_list()) + async def _async_send_ctrl_msg(self, msg): + await self._async_send_msg(self.ws_ctrl_add, msg) def send_ctrl_msg(self, msg): """Send control message @@ -221,7 +266,7 @@ def send_ctrl_msg(self, msg): Args: msg (str): Message to send to the Bela watcher. Example: {"watcher": [{"cmd": "list"}]} """ - self._send_msg(self.ws_ctrl_add, json.dumps(msg)) + self._send_msg(self.ws_ctrl_add, msg) # --- private methods --- # @@ -235,7 +280,7 @@ async def _async_start_listener(self, ws, ws_address): ws_address (str): Websocket address """ try: - while ws is not None and ws.open: + while ws is not None and ws.state == 1: msg = await ws.recv() if self._printall_responses: print(msg) @@ -248,12 +293,23 @@ async def _async_start_listener(self, ws, ws_address): self._list_response_queue.put_nowait(_msg["watcher"]) else: print(msg) - except Exception as e: - if ws.open: # otherwise websocket was closed intentionally + + except Exception or asyncio.exceptions.CancelledError as e: + if ws.state == 1: # otherwise websocket was closed intentionally _handle_connection_exception( ws_address, e, "receiving message") # send message + async def _async_send_msg(self, ws_address, msg): + try: + if ws_address == self.ws_data_add and self.ws_data is not None and self.ws_data.state == 1: + await self._to_send_data_msg_queue.put(msg) + elif ws_address == self.ws_ctrl_add and self.ws_ctrl is not None and self.ws_ctrl.state == 1: + # msg = json.dumps(msg) + await self._to_send_ctrl_msg_queue.put(msg) + except Exception as e: + _handle_connection_exception(ws_address, e, "sending message") + return 0 def _send_msg(self, ws_address, msg): """Send message to websocket @@ -262,27 +318,27 @@ def _send_msg(self, ws_address, msg): ws_address (str): Websocket address msg (str): Message to send """ - async def _async_send_msg(ws_address, msg): - try: - if ws_address == self.ws_data_add and self.ws_data is not None and self.ws_data.open: - asyncio.create_task(self._to_send_data_msg_queue.put(msg)) - elif ws_address == self.ws_ctrl_add and self.ws_ctrl is not None and self.ws_ctrl.open: - await self.ws_ctrl.send(msg) - except Exception as e: - _handle_connection_exception(ws_address, e, "sending message") - return 0 - asyncio.run(_async_send_msg(ws_address, msg)) + return self.loop.create_task(self._async_send_msg(ws_address, msg)) # send messages async def _send_data_msg_worker(self): """ Send data message to websocket. Runs as long as websocket is open. """ - while self.ws_data is not None and self.ws_data.open: + while self.ws_data is not None and self.ws_data.state == 1: msg = await self._to_send_data_msg_queue.get() await self.ws_data.send(msg) self._to_send_data_msg_queue.task_done() + async def _send_ctrl_msg_worker(self): + """ Send control message to websocket. Runs as long as websocket is open. + """ + while self.ws_ctrl is not None and self.ws_ctrl.state == 1: + msg = await self._to_send_ctrl_msg_queue.get() + msg = json.dumps(msg) + await self.ws_ctrl.send(msg) + self._to_send_ctrl_msg_queue.task_done() + # process messages async def _process_data_msg_worker(self): @@ -292,7 +348,7 @@ async def _process_data_msg_worker(self): msg (str): Bytestring with data """ - while self.ws_data is not None and self.ws_data.open: + while self.ws_data is not None and self.ws_data.state == 1: msg = await self._received_data_msg_queue.get() await self._process_data_msg(msg) self._received_data_msg_queue.task_done() @@ -377,6 +433,13 @@ def _parse_binary_data(self, binary_data, timestamp_mode, _type): # --- utils --- # + def wait(self, time_in_seconds): + self.loop.run_until_complete(asyncio.sleep(time_in_seconds)) + + async def _async_get_latest_timestamp(self): + _list = await self._async_list() + return _list["timestamp"] + def get_latest_timestamp(self): return self.list()["timestamp"] @@ -533,16 +596,44 @@ def get_buffer_size(self, var_type, timestamp_mode): # return error message return 0 - def _cancel_tasks(self, tasks): + async def _async_cancel_tasks(self, tasks): """Cancels tasks """ + cancel_tasks = [] for task in tasks: - if task is not None: + if task is not None and not task.done(): task.cancel() + cancel_tasks.append(task) + await asyncio.gather(*cancel_tasks, return_exceptions=True) + + async def _async_cleanup(self): + """Cleans up tasks + """ + tasks = [self._ctrl_listener_task, + self._data_listener_task, + self._process_received_data_msg_task, + self._send_data_msg_task, + self._send_ctrl_msg_task + ] + await self._async_cancel_tasks(tasks) + await self._async_disconnect() + + def cleanup(self): + """Cleans up tasks. Synchronous wrapper for _async_cleanup + """ + self.loop.run_until_complete(self._async_cleanup()) # destructor + def __del__(self): - self.disconnect() # stop websockets + pass + # self.disconnect() # stop websockets and cancel tasks + # stop event loop + # self.wait(0.5) + # if self.loop.is_running(): + # self.loop.stop() + # self.loop.close() + # self.loop_thread.join() def _handle_connection_exception(ws_address, exception, action): diff --git a/requirements.txt b/requirements.txt index 1acd0ce..aa9d1f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ notebook==7.2.2 numpy==1.26.0 panel==0.14.4 paramiko==3.5.0 -websockets==12.0 +websockets==14.1 diff --git a/test/test-send.py b/test/test-send.py index 3281dfe..aedd412 100644 --- a/test/test-send.py +++ b/test/test-send.py @@ -1,16 +1,11 @@ import unittest from pybela import Streamer -import struct import numpy as np -import asyncio streamer = Streamer() variables = ["myvar1", "myvar2"] -async def wait(): - await asyncio.sleep(0.1) - # can't be merged with test.py because in the render.cpp the watcher needs to be 'ticked' when iterating the buffer, not at every audio frame! # TOOD test other types (int, double, uint, char) @@ -27,12 +22,12 @@ def test_send_buffer(self): for id in [0, 1]: # buffers are only sent from Bela to the host once full, so it needs to be 1024 long to be sent - buffer_id, buffer_type, buffer_length, empty = id, 'f', 1024, 0 + buffer_id, buffer_type, buffer_length = id, 'f', 1024 data_list = np.arange(1, buffer_length+1, 1) streamer.send_buffer(buffer_id, buffer_type, buffer_length, data_list) - asyncio.run(wait()) # wait for the buffer to be sent + streamer.wait(0.1) # wait for the buffer to be sent for var in variables: assert np.array_equal( diff --git a/test/test.py b/test/test.py index a900c94..32302a3 100644 --- a/test/test.py +++ b/test/test.py @@ -1,10 +1,9 @@ import unittest -import asyncio import os import numpy as np from pybela import Watcher, Streamer, Logger, Monitor, Controller -os.environ["PYTHONASYNCIODEBUG"] = "1" +# os.environ["PYTHONASYNCIODEBUG"] = "1" # all tests should be run with Bela connected and the bela-test project (in test/bela-test) running on the board @@ -16,7 +15,8 @@ def setUp(self): self.watcher.connect() def tearDown(self): - self.watcher.__del__() + self.watcher.cleanup() + # pass def test_list(self): self.assertEqual(len(self.watcher.list()["watchers"]), len(self.watcher.watcher_vars), @@ -46,7 +46,7 @@ def setUp(self): self.saving_filename = "test_streamer_save.txt" def tearDown(self): - self.streamer.__del__() + self.streamer.cleanup() def test_stream_n_values(self): n_values = 40 @@ -109,7 +109,8 @@ def test_start_stop_streaming(self): # check streaming mode is FOREVER after start_streaming is called self.assertEqual(self.streamer._streaming_mode, "FOREVER", "Streaming mode should be FOREVER after start_streaming") - asyncio.run(asyncio.sleep(0.5)) # wait for some data to be streamed + # wait for some data to be streamed + self.streamer.wait(0.5) self.streamer.stop_streaming(variables=self.streaming_vars) # check streaming mode is OFF after stop_streaming @@ -151,7 +152,7 @@ def callback(buffer): self.streamer.start_streaming( variables, saving_enabled=False, on_buffer_callback=callback) - asyncio.run(asyncio.sleep(0.1)) + self.streamer.wait(0.5) self.streamer.stop_streaming(variables) @@ -175,7 +176,7 @@ def callback(block): self.streamer.start_streaming( variables, saving_enabled=False, on_block_callback=callback) - asyncio.run(asyncio.sleep(0.5)) + self.streamer.wait(0.5) self.streamer.stop_streaming(variables) @@ -203,7 +204,7 @@ def setUp(self): self.logging_dir = "./test" def tearDown(self): - self.logger.__del__() + self.logger.cleanup() def _test_logged_data(self, logger, logging_vars, local_paths): # common routine to test the data in the logged files @@ -234,7 +235,7 @@ def test_logged_files_with_transfer(self): # log with transfer file_paths = self.logger.start_logging( variables=self.logging_vars, transfer=True, logging_dir=self.logging_dir) - asyncio.run(asyncio.sleep(0.5)) + self.logger.wait(0.5) self.logger.stop_logging() # test logged data @@ -252,7 +253,7 @@ def test_logged_files_wo_transfer(self): # logging without transfer file_paths = self.logger.start_logging( variables=self.logging_vars, transfer=False, logging_dir=self.logging_dir) - asyncio.run(asyncio.sleep(0.5)) + self.logger.wait(0.5) self.logger.stop_logging() # transfer files from bela @@ -313,7 +314,7 @@ def setUp(self): self.monitor.connect() def tearDown(self): - self.monitor.__del__() + self.monitor.cleanup() def test_peek(self): peeked_values = self.monitor.peek() # peeks at all variables by default @@ -325,7 +326,7 @@ def test_period_monitor(self): self.monitor.start_monitoring( variables=self.monitor_vars[:2], periods=[self.period]*len(self.monitor_vars[:2])) - asyncio.run(asyncio.sleep(0.5)) + self.monitor.wait(0.5) monitored_values = self.monitor.stop_monitoring() for var in self.monitor_vars[:2]: # assigned at every frame n @@ -364,7 +365,7 @@ def test_save_monitor(self): saving_enabled=True, saving_filename=self.saving_filename, saving_dir=self.saving_dir) - asyncio.run(asyncio.sleep(0.5)) + self.monitor.wait(0.5) monitored_buffers = self.monitor.stop_monitoring() for var in self.monitor_vars: @@ -389,7 +390,7 @@ def setUp(self): self.controller.connect() def tearDown(self): - self.controller.__del__() + self.controller.cleanup() def test_start_stop_controlling(self): self.controller.start_controlling(variables=self.controlled_vars) @@ -410,7 +411,7 @@ def test_send_value(self): self.controller.send_value( variables=self.controlled_vars, values=[set_value]*len(self.controlled_vars)) - asyncio.run(asyncio.sleep(0.1)) # wait for the values to be set + self.controller.wait(0.1) # wait for the values to be set _controlled_values = self.controller.get_value( variables=self.controlled_vars) # avoid multiple calls to list From 362cf72ce0ec5b20fb48662a2cc96baf39b5528b Mon Sep 17 00:00:00 2001 From: Teresa Pelinski Date: Mon, 16 Dec 2024 20:16:28 +0000 Subject: [PATCH 3/5] passing all tests --- pybela/Logger.py | 147 +++++++++-------- pybela/Monitor.py | 13 +- pybela/Streamer.py | 395 +++++++++++++++++++++++---------------------- pybela/Watcher.py | 289 +++++++++++++++++---------------- test/test.py | 11 +- 5 files changed, 454 insertions(+), 401 deletions(-) diff --git a/pybela/Logger.py b/pybela/Logger.py index 10a487f..1014559 100644 --- a/pybela/Logger.py +++ b/pybela/Logger.py @@ -4,7 +4,7 @@ import struct import paramiko from .Watcher import Watcher -from .utils import _bcolors, _print_error, _print_info, _print_ok, _print_warning +from .utils import _print_error, _print_info, _print_ok, _print_warning class Logger(Watcher): @@ -30,6 +30,8 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add self._mode = "LOG" + # -- logging methods -- + def start_logging(self, variables=[], transfer=True, logging_dir="./"): """ Starts logging session. The session can be ended by calling stop_logging(). @@ -45,6 +47,7 @@ def start_logging(self, variables=[], transfer=True, logging_dir="./"): local_paths = {} if transfer: + self.connect_ssh() # start ssh connection for var in [v for v in self.watcher_vars if v["name"] in variables]: var = var["name"] local_path = os.path.join( @@ -86,6 +89,8 @@ async def _async_schedule_logging(variables, timestamps, durations, transfer, lo local_paths = {} if transfer: + self.connect_ssh() # start ssh connection + async def _async_check_if_file_exists_and_start_copying(var, timestamp): diff_stamps = timestamp - latest_timestamp @@ -135,7 +140,8 @@ async def _async_check_if_file_exists_and_start_copying(var, timestamp): await asyncio.gather(*self._active_copying_tasks, return_exceptions=True) self._active_copying_tasks.clear() _active_checking_tasks.clear() - self.sftp_client.close() + if self.sftp_client: + self.disconnect_ssh() # async version (non blocking) # async def _async_cleanup(): @@ -158,7 +164,7 @@ async def __async_logging_common_routine(self, mode, timestamps=[], durations=[] if self.is_logging(): self.loop.create_task(self._async_stop_logging()) - self.connect_ssh() # start ssh connection + # self.connect_ssh() # start ssh connection self._logging_mode = mode @@ -178,23 +184,28 @@ async def __async_logging_common_routine(self, mode, timestamps=[], durations=[] return remote_paths async def _async_stop_logging(self, variables=[]): + """Stops logging session. + + Args: + variables (list of str, optional): List of variables to stop logging. If none is passed, logging is stopped for all variables in the watcher. Defaults to []. + """ self._logging_mode = "OFF" if variables == []: # if no variables specified, stop streaming all watcher variables (default) variables = [var["name"] for var in self.watcher_vars] - self.send_ctrl_msg( + await self._async_send_ctrl_msg( {"watcher": [{"cmd": "unlog", "watchers": variables}]}) _print_info(f"Stopped logging variables {variables}...") await asyncio.gather(*self._active_copying_tasks, return_exceptions=True) self._active_copying_tasks.clear() - - self.sftp_client.close() + if self.sftp_client: + self.disconnect_ssh() def stop_logging(self, variables=[]): - """ Stops logging session. + """ Stops logging session. Sync wrapper for _async_stop_logging(). Args: variables (list of str, optional): List of variables to stop logging. If none is passed, logging is stopped for all variables in the watcher. Defaults to []. @@ -202,57 +213,21 @@ def stop_logging(self, variables=[]): return self.loop.run_until_complete(self._async_stop_logging(variables)) - def connect_ssh(self): - """ Connects to Bela via ssh to transfer log files. - """ - - if self.sftp_client is not None: - self.disconnect_ssh() - - self.ssh_client = paramiko.SSHClient() - self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - - # Workaround for no authentication: - # https://github.com/paramiko/paramiko/issues/890#issuecomment-906893725 - try: - self.ssh_client.connect( - self.ip, port=22, username="root", password=None) - except paramiko.SSHException as e: - self.ssh_client.get_transport().auth_none("root") - except Exception as e: - _print_error( - f"Error while connecting to Bela via ssh: {e} {bcolors.ENDC}") - return - - self.sftp_client = self.ssh_client.open_sftp() # TODO handle exceptions better - - def disconnect_ssh(self): - """ Disconnects from Bela via ssh. - """ - if self.sftp_client: - self.sftp_client.close() - if self.ssh_client: - self.ssh_client.close() - - def is_logging(self): - """ Returns True if the logger is currently logging, false otherwise. - - Returns: - bool: Logger status - """ - return True if self._logging_mode != "OFF" else False + # -- binary file parsing method def read_binary_file(self, file_path, timestamp_mode): """ Reads a binary file generated by the logger and returns a dictionary with the file contents. Args: file_path (str): Path of the file to be read. - timestamp_mode (str): Timestamp mode of the variable. Can be "dense" or "sparse". + timestamp_mode (str): Timestamp mode of the variable. Can be "dense" or "sparse". Returns: dict: Dictionary with the file contents. """ - + if file_path is None: + _print_error("Error: No file path provided.") + return file_size = os.path.getsize(file_path) assert file_size != 0, f"Error: The size of {file_path} is 0." @@ -308,9 +283,51 @@ def _parse_null_terminated_string(file): "buffers": parsed_buffers } - # - utils + # -- ssh methods & utils -- - # -- ssh copy utils + def connect_ssh(self): + """ Connects to Bela via ssh to transfer log files. + """ + + if self.sftp_client is not None: + self.disconnect_ssh() + + self.ssh_client = paramiko.SSHClient() + self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + # Workaround for no authentication: + # https://github.com/paramiko/paramiko/issues/890#issuecomment-906893725 + try: + self.ssh_client.connect( + self.ip, port=22, username="root", password=None) + except paramiko.SSHException as e: + self.ssh_client.get_transport().auth_none("root") + except Exception as e: + _print_error( + f"Error while connecting to Bela via ssh: {e} {bcolors.ENDC}") + return + + try: + self.sftp_client = self.ssh_client.open_sftp() + # _print_ok("SSH connection and SFTP client established successfully.") + except Exception as e: + _print_error( + f"Error while opening SFTP client: {e} {bcolors.ENDC}") + self.disconnect_ssh() + + def disconnect_ssh(self): + """ Disconnects from Bela via ssh. + """ + if self.sftp_client: + self.sftp_client.close() + + def is_logging(self): + """ Returns True if the logger is currently logging, false otherwise. + + Returns: + bool: Logger status + """ + return True if self._logging_mode != "OFF" else False def __copy_file_in_chunks(self, remote_path, local_path, chunk_size=2**12): """ Copies a file from the remote path to the local path in chunks. This function is called by start_logging() if transfer=True. @@ -380,9 +397,10 @@ def copy_file_from_bela(self, remote_path, local_path, verbose=True): verbose (bool, optional): Show info messages. Defaults to True. """ self.connect_ssh() - self.loop.run_until_complete(self._async_copy_file_from_bela( + local_path = self.loop.run_until_complete(self._async_copy_file_from_bela( remote_path, local_path, verbose)) self.disconnect_ssh() + return local_path def copy_all_bin_files_in_project(self, dir="./", verbose=True): """ Copies all .bin files in the specified remote directory using SFTP. @@ -418,23 +436,28 @@ async def _async_copy_file_from_bela(self, remote_path, local_path, verbose=Fals local_path (str): Path to the file in the local machine (where the file is copied to) """ try: + _local_path = None if os.path.exists(local_path): - local_path = self._generate_local_filename(local_path) + _local_path = self._generate_local_filename(local_path) + else: + _local_path = local_path transferred_event = asyncio.Event(loop=self.loop) def callback(transferred, to_transfer): return transferred_event.set( ) if transferred == to_transfer else None - self.sftp_client.get(remote_path, local_path, callback=callback) - await asyncio.wait_for(transferred_event.wait(), timeout=3) + self.sftp_client.get(remote_path, _local_path, callback=callback) + file_size = self.sftp_client.stat(remote_path).st_size + await asyncio.wait_for(transferred_event.wait(), timeout=file_size/1e5) if verbose: _print_ok( - f"\rTransferring {remote_path}-->{local_path}... Done.") - return transferred_event.is_set() - except asyncio.TimeoutError: - _print_error("Timeout while transferring file.") - return False # File copy did not complete within the timeout + f"\rTransferring {remote_path}-->{_local_path}... Done.") + return local_path + except asyncio.exceptions.TimeoutError: + _print_error( + f"Error while transferring file: TimeoutError.") + return None except Exception as e: _print_error(f"Error while transferring file: {e}") - return False + return None def finish_copying_file(self, remote_path, local_path): # TODO test """Finish copying file if it was interrupted. This function is used to copy the remaining part of a file that was interrupted during the copy process. @@ -483,8 +506,6 @@ def finish_copying_file(self, remote_path, local_path): # TODO test self.disconnect_ssh() - # -- ssh delete utils - def delete_file_from_bela(self, remote_path, verbose=True): """Deletes a file from the remote path in Bela. @@ -543,8 +564,6 @@ async def _async_delete_file_from_bela(self, remote_path, verbose=True): f"Error while deleting file in Bela: {e} ") break - # -- bunk task utils - def _action_on_all_bin_files_in_project(self, action, local_dir=None): # List all files in the remote directory remote_path = f'/root/Bela/projects/{self.project_name}' diff --git a/pybela/Monitor.py b/pybela/Monitor.py index 846910a..750a437 100644 --- a/pybela/Monitor.py +++ b/pybela/Monitor.py @@ -81,10 +81,10 @@ def start_monitoring(self, variables=[], periods=[], saving_enabled=False, savin """ variables = self._var_arg_checker(variables) - periods = self._check_periods(periods, variables) + self._periods = self._check_periods(periods, variables) self.start_streaming( - variables=variables, periods=periods, saving_enabled=saving_enabled, saving_filename=saving_filename, saving_dir=saving_dir) + variables=variables, periods=self._periods, saving_enabled=saving_enabled, saving_filename=saving_filename, saving_dir=saving_dir) def monitor_n_values(self, variables=[], periods=[], n_values=1000, saving_enabled=False, saving_filename="monitor.txt"): """ @@ -109,8 +109,8 @@ def monitor_n_values(self, variables=[], periods=[], n_values=1000, saving_enabl monitored_buffers_queue (dict): Dict containing the monitored buffers for each streamed variable. """ variables = self._var_arg_checker(variables) - periods = self._check_periods(periods, variables) - self.stream_n_values(variables, periods, n_values, + self._periods = self._check_periods(periods, variables) + self.stream_n_values(variables, self._periods, n_values, saving_enabled, saving_filename) return self.values @@ -133,14 +133,15 @@ async def async_monitor_n_values(self, variables=[], periods=[], n_values=1000, deque: Monitored buffers queue """ variables = self._var_arg_checker(variables) - periods = self._check_periods(periods, variables) + self._periods = self._check_periods(periods, variables) - self.async_stream_n_values(variables, periods, n_values, + self.async_stream_n_values(variables, self._periods, n_values, saving_enabled, saving_filename) async def _async_stop_monitoring(self, variables=[]): await self._async_stop_streaming(variables) self._monitored_vars = None + self._periods = None return {var: self.values[var] for var in self.values if self.values[var]["timestamps"] != []} def stop_monitoring(self, variables=[]): diff --git a/pybela/Streamer.py b/pybela/Streamer.py index b481427..4ef1052 100644 --- a/pybela/Streamer.py +++ b/pybela/Streamer.py @@ -18,6 +18,8 @@ from .Watcher import Watcher from .utils import _print_info, _print_error, _print_warning +import numpy as np + class Streamer(Watcher): def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add="gui_control"): @@ -59,16 +61,15 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add self._monitored_vars = None self._peek_response_available = asyncio.Event(loop=self.loop) self._peek_response = None + self._periods = None self._mode = "STREAM" - # --- public methods --- # - - # - setters & getters + # -- properties -- @property def monitored_vars(self): - """ Returns a list of monitored variables. If no variables are monitored, returns an empty list. + """ Returns a list of monitored variables. If no variables are monitored, returns an empty list. Can't be used in async functions, use _async_monitored_vars() instead. Returns: list: list of monitored variables @@ -82,6 +83,11 @@ def monitored_vars(self): return self._monitored_vars async def _async_monitored_vars(self): + """Async version of monitored_vars. Returns a list of monitored variables. If no variables are monitored, returns an empty list. + + Returns: + _type_: _description_ + """ if self._monitored_vars is None: _list = await self._async_list() self._monitored_vars = self._filtered_watcher_vars( @@ -131,7 +137,7 @@ def streaming_buffers_data(self): "MONITOR" else [_buffer["value"]]) return data - # - streaming methods + # -- streaming methods -- def __streaming_common_routine(self, variables=[], saving_enabled=False, saving_filename="var_stream.txt", saving_dir="./", on_buffer_callback=None, on_block_callback=None, callback_args=()): @@ -140,11 +146,16 @@ def __streaming_common_routine(self, variables=[], saving_enabled=False, saving_ self.stop_streaming() # stop any previous streaming if not self.is_connected(): - _print_warning( + + raise ConnectionError( f'{"Monitor" if self._mode=="MONITOR" else "Streamer" } is not connected to Bela. Run {"monitor" if self._mode=="MONITOR" else "streamer"}.connect() first.') - return 0 + + # reset streaming buffers queue self._streaming_buffers_queue = {var["name"]: deque( maxlen=self._streaming_buffers_queue_length) for var in self.watcher_vars} + # clear asyncio data queues + self._processed_data_msg_queue = asyncio.Queue(loop=self.loop) + self.last_streamed_buffer = { var["name"]: {"data": [], "timestamps": []} for var in self.watcher_vars} @@ -155,9 +166,6 @@ def __streaming_common_routine(self, variables=[], saving_enabled=False, saving_ self._saving_filename = self._generate_filename( saving_filename, saving_dir) if saving_enabled else None - self._processed_data_msg_queue = asyncio.Queue( - loop=self.loop) # clear processed data queue - async def async_callback_workers(): if on_block_callback and on_buffer_callback: @@ -181,9 +189,9 @@ async def async_callback_workers(): def start_streaming(self, variables=[], periods=[], saving_enabled=False, saving_filename="var_stream.txt", saving_dir="./", on_buffer_callback=None, on_block_callback=None, callback_args=()): """ - Starts the streaming session. The session can be stopped with stop_streaming(). + Starts the streaming session. The session can be stopped with stop_streaming(). Can't be used in async functions. - If no variables are specified, all watcher variables are streamed. If saving_enabled is True, the streamed data is saved to a local file. If saving_filename is None, the default filename is used with the variable name appended to its start. The filename is automatically incremented if it already exists. + If no variables are specified, all watcher variables are streamed. If saving_enabled is True, the streamed data is saved to a local file. If saving_filename is None, the default filename is used with the variable name appended to its start. The filename is automatically incremented if it already exists. Args: variables (list, optional): List of variables to be streamed. Defaults to []. @@ -198,7 +206,7 @@ def start_streaming(self, variables=[], periods=[], saving_enabled=False, saving variables = self.__streaming_common_routine( variables, saving_enabled, saving_filename, saving_dir, on_buffer_callback, on_block_callback, callback_args) - + _all_vars = [var["name"] for var in self.watcher_vars] # commented because then you can only start streaming on variables whose values have been previously assigned in the Bela code # not useful for the Sender function (send a buffer from the laptop and stream it through the watcher) # async def async_wait_for_streaming_to_start(): # ensures that when function returns streaming has started @@ -214,11 +222,8 @@ def start_streaming(self, variables=[], periods=[], saving_enabled=False, saving if periods != []: warnings.warn( "Periods list is ignored in streaming mode STREAM") - self._to_send_ctrl_msg_queue.put_nowait( - {"watcher": [{"cmd": "watch", "watchers": variables}]}) - # self.send_ctrl_msg( - # {"watcher": [{"cmd": "watch", "watchers": variables}]}) - # # asyncio.run(async_wait_for_streaming_to_start()) + self.send_ctrl_msg( + {"watcher": [{"cmd": "watch", "watchers": variables, "periods": [0]*len(_all_vars)}]}) _print_info( f"Started streaming variables {variables}... Run stop_streaming() to stop streaming.") elif self._mode == "MONITOR": @@ -233,6 +238,11 @@ def start_streaming(self, variables=[], periods=[], saving_enabled=False, saving _print_info(f"Peeking at variables {variables}...") async def _async_stop_streaming(self, variables=[]): + """ Stops the current streaming session for the given variables. If no variables are passed, the streaming of all variables is interrupted. + + Args: + variables (list, optional): _description_. Defaults to []. + """ _previous_streaming_mode = copy.copy(self._streaming_mode) self._streaming_mode = "OFF" @@ -244,31 +254,34 @@ async def _async_stop_streaming(self, variables=[]): await asyncio.gather(*self._active_saving_tasks, return_exceptions=True, loop=self.loop) self._active_saving_tasks.clear() + _all_vars = [var["name"] for var in self.watcher_vars] if variables == []: # if no variables specified, stop streaming all watcher variables (default) - variables = [var["name"] for var in self.watcher_vars] + variables = _all_vars if self._mode == "STREAM" and _previous_streaming_mode != "SCHEDULE": - self.send_ctrl_msg( + await self._async_send_ctrl_msg( {"watcher": [{"cmd": "unwatch", "watchers": variables}]}) _print_info(f"Stopped streaming variables {variables}...") - elif self._mode == "MONITOR" and _previous_streaming_mode != "SCHEDULE": - self.send_ctrl_msg( - {"watcher": [{"cmd": "monitor", "periods": [0]*len(variables), "watchers": variables}]}) # setting period to 0 disables monitoring + # elif self._mode == "MONITOR" and _previous_streaming_mode != "SCHEDULE": + elif self._mode == "MONITOR": + await self._async_send_ctrl_msg( + {"watcher": [{"cmd": "monitor", "periods": [0]*len(_all_vars), "watchers": variables}]}) # setting period to 0 disables monitoring if not _previous_streaming_mode == "PEEK": _print_info(f"Stopped monitoring variables {variables}...") - self._processed_data_msg_queue = asyncio.Queue( - loop=self.loop) # clear processed data queue - self._on_buffer_callback_is_active = False - if self._on_buffer_callback_worker_task: - await self._on_buffer_callback_worker_task.cancel() - self._on_block_callback_is_active = False - if self._on_block_callback_worker_task: - await self._on_block_callback_worker_task.cancel() + + self._processed_data_msg_queue = asyncio.Queue( + loop=self.loop) # clear processed data queue + self._on_buffer_callback_is_active = False + if self._on_buffer_callback_worker_task: + self._on_buffer_callback_worker_task.cancel() + self._on_block_callback_is_active = False + if self._on_block_callback_worker_task: + self._on_block_callback_worker_task.cancel() def stop_streaming(self, variables=[]): """ - Stops the current streaming session for the given variables. If no variables are passed, the streaming of all variables is interrupted. + Stops the current streaming session for the given variables. If no variables are passed, the streaming of all variables is interrupted. Sync wrapper of _async_stop_streaming(), can't be used in async functions. Args: variables (list, optional): List of variables to stop streaming. Defaults to []. @@ -329,12 +342,13 @@ async def async_check_if_variables_have_been_streamed_and_stop(): def stream_n_values(self, variables=[], periods=[], n_values=1000, saving_enabled=False, saving_filename=None, saving_dir="./", on_buffer_callback=None, on_block_callback=None, callback_args=()): """ - Streams a given number of values. Since the data comes in buffers of a predefined size, always an extra number of frames will be streamed (unless the number of frames is a multiple of the buffer size). + Streams a given number of values. Since the data comes in buffers of a predefined size, always an extra number of frames will be streamed (unless the number of frames is a multiple of the buffer size). Note: This function will block the main thread until n_values have been streamed. Since the streamed values come in blocks, the actual number of returned frames streamed may be higher than n_values, unless n_values is a multiple of the block size (streamer._streaming_block_size). To avoid blocking, use the async version of this function: - stream_task = self.loop.create_task(streamer.async_stream_n_values(variables, n_values, periods, saving_enabled, saving_filename)) + stream_task = self.loop.create_task(streamer.async_stream_n_values( + variables, n_values, periods, saving_enabled, saving_filename)) and retrieve the streaming buffer using: streaming_buffers_queue = await stream_task @@ -356,9 +370,10 @@ def stream_n_values(self, variables=[], periods=[], n_values=1000, saving_enable return self.loop.run_until_complete(self.async_stream_n_values(variables, periods, n_values, saving_enabled, saving_filename, saving_dir, on_buffer_callback, on_block_callback, callback_args)) async def async_stream_n_values(self, variables=[], periods=[], n_values=1000, saving_enabled=False, saving_filename="var_stream.txt", saving_dir="./", on_buffer_callback=None, on_block_callback=None, callback_args=()): - """ - Asynchronous version of stream_n_values(). Usage: - stream_task = self.loop.create_task(streamer.async_stream_n_values(variables, n_values, saving_enabled, saving_filename)) + """ + Asynchronous version of stream_n_values(). Usage: + stream_task = self.loop.create_task(streamer.async_stream_n_values( + variables, n_values, saving_enabled, saving_filename)) and retrieve the streaming buffer using: streaming_buffers_queue = await stream_task @@ -388,9 +403,10 @@ async def async_stream_n_values(self, variables=[], periods=[], n_values=1000, s # if mode stream, each buffer has m values and we need to calc the min buffers needed to supply n_values # variables might have different buffer sizes -- the code below finds the minimum number of buffers needed to stream n_values for all variables + _watcher_vars = await self._async_watcher_vars() buffer_sizes = [ self.get_data_length(var["type"], var["timestamp_mode"]) - for var in self.watcher_vars if var["name"] in variables] + for var in _watcher_vars if var["name"] in variables] # TODO add a warning when there's different buffer sizes ? @@ -402,7 +418,7 @@ async def async_stream_n_values(self, variables=[], periods=[], n_values=1000, s if periods != []: warnings.warn( "Periods list is ignored in streaming mode STREAM") - self._to_send_ctrl_msg_queue.put_nowait( + await self._async_send_ctrl_msg( {"watcher": [{"cmd": "unwatch", "watchers": [var["name"] for var in self.watcher_vars]}, {"cmd": "watch", "watchers": variables}]}) _print_info( f"Streaming {n_values} values for variables {variables}...") @@ -412,7 +428,7 @@ async def async_stream_n_values(self, variables=[], periods=[], n_values=1000, s self.streaming_buffers_queue_length = n_values periods = self._check_periods(periods, variables) - self._to_send_ctrl_msg_queue.put_nowait( + await self._async_send_ctrl_msg( {"watcher": [{"cmd": "monitor", "watchers": variables, "periods": periods}]}) _print_info( f"Monitoring {n_values} values for variables {variables} with periods {periods}...") @@ -428,7 +444,108 @@ async def async_stream_n_values(self, variables=[], periods=[], n_values=1000, s return self.streaming_buffers_queue - # callbacks + # -- data processing method -- + + async def _process_data_msg(self, msg): + """ Process data message received from Bela. This function is called by the websocket listener when a data message is received. + + Args: + msg (bytestring): Data message received from Bela + """ + + global _channel, _type + try: + _, __ = _channel, _type + except NameError: # initialise global variables to None + _channel = None + _type = None + + # in case buffer is received whilst streaming mode is on but parsed after streaming_enabled has changed + _saving_enabled = copy.copy(self._saving_enabled) + if self._streaming_mode != "OFF": + if len(msg) in [3, 4]: # _channel can be either 1 or 2 bytes long + # parse buffer header + _channel, _type = re.search( + r'(\d+).*?(\w)', msg.decode()).groups() + _channel = int(_channel) + + assert _type in ['i', 'f', 'j', 'd', + 'c'], f"Unsupported type: {_type}" + + assert _type == self._watcher_vars[_channel][ + 'type'], f"Type mismatch: {_type} != {self._watcher_vars[_channel]['type']}" + + # convert unsigned int to int -- struct does not support unsigned ints + _type = 'i' if _type == 'j' else _type + + elif len(msg) > 3 and _channel is not None and _type is not None: + var_name = self._watcher_vars[_channel]['name'] + var_timestamp_mode = self._watcher_vars[_channel]["timestamp_mode"] + + # parse buffer body + parsed_buffer = self._parse_binary_data( + msg, var_timestamp_mode, _type).copy() + + # put in processed_queue if callback is true + if self._on_buffer_callback_is_active or self._on_block_callback_is_active: + await self._processed_data_msg_queue.put({"name": var_name, "buffer": parsed_buffer}) + + # fixes bug where data is shifted by period + _var_streaming_buffers_queue = copy.copy( + self._streaming_buffers_queue[var_name]) + _var_streaming_buffers_queue.append(parsed_buffer) + self._streaming_buffers_queue[var_name] = _var_streaming_buffers_queue + + # populate last streamed buffer + if self._mode == "STREAM": + self.last_streamed_buffer[var_name]["data"] = parsed_buffer["data"] + if var_timestamp_mode == "dense": + self.last_streamed_buffer[var_name]["timestamps"] = [ + parsed_buffer["ref_timestamp"] + i for i in range(0, len(parsed_buffer["data"]))] + elif var_timestamp_mode == "sparse": # sparse + self.last_streamed_buffer[var_name]["timestamps"] = [ + parsed_buffer["ref_timestamp"] + i for i in parsed_buffer["rel_timestamps"]] + elif self._mode == "MONITOR": + self.last_streamed_buffer[var_name] = { + "timestamp": parsed_buffer["timestamp"], "value": parsed_buffer["value"]} + # save data to file if saving is enabled + if _saving_enabled: + _saving_var_filename = os.path.join(os.path.dirname( + self._saving_filename), f"{var_name}_{os.path.basename(self._saving_filename)}") + # save the data asynchronously + saving_task = self.loop.create_task( + self._save_data_to_file(_saving_var_filename, parsed_buffer)) + self._active_saving_tasks.append(saving_task) + + # response to .peek() call + if self._mode == "MONITOR" and self._peek_response is not None: + # check that all the watched variables have been received + self._peek_response[self._watcher_vars[_channel]["name"]] = { + "timestamp": parsed_buffer["timestamp"], "value": parsed_buffer["value"]} + # notify peek() that data is available + if all(value is not None for value in self._peek_response.values()): + self._peek_response_available.set() + + # if streaming buffers queue is full for watched variables and streaming mode is n_values + if self._streaming_mode == "N_VALUES": # FIXME doesn't always work for monitor + _watched_vars = await self._async_watched_vars() + _monitored_vars = await self._async_monitored_vars() + _vars = _watched_vars if self._mode == "STREAM" else _monitored_vars + if all(len(self._streaming_buffers_queue[var["name"]]) == self._streaming_buffers_queue_length + for var in _vars): + + # check if timestamp values are spaced by the correct period + if self._mode == "MONITOR" and np.any([np.diff(self.values[var["name"]]["timestamps"]) != self._periods[idx] for idx, var in enumerate(_monitored_vars)]): + for var in _monitored_vars: + # fixes bug in which the diff between first and second timestamp is less than period + self._streaming_buffers_queue[var["name"]].popleft( + ) + + else: + self._streaming_mode = "OFF" + self._streaming_buffer_available.set() + + # -- callback methods -- async def __async_on_buffer_callback_worker(self, on_buffer_callback, callback_args): while self._on_buffer_callback_is_active and self.is_streaming(): @@ -487,7 +604,7 @@ async def __async_on_block_callback_worker(self, on_block_callback, callback_arg await asyncio.sleep(0.001) - # send + # -- data sending methods -- def send_buffer(self, buffer_id, buffer_type, buffer_length, data_list, verbose=False): """ @@ -511,49 +628,7 @@ def send_buffer(self, buffer_id, buffer_type, buffer_length, data_list, verbose= _print_info( f"Sent buffer {buffer_id} of type {buffer_type} with length {buffer_length}...") - # - utils - - def is_streaming(self): - """Returns True if the streamer is currently streaming, False otherwise. - - Returns: - bool: Streaming status bool - """ - return True if self._streaming_mode != "OFF" else False - - def flush_queue(self): - """Flushes the streaming buffers queue. The queue is emptied and the insertion counts are reset to 0. - """ - self._streaming_buffers_queue = {var["name"]: deque( - maxlen=self._streaming_buffers_queue_length) for var in self.watcher_vars} - - def load_data_from_file(self, filename): - """ - Loads data from a file saved through the saving_enabled function in start_streaming() or stream_n_values(). The file should contain a list of dicts, each dict containing a variable name and a list of values. The list of dicts should be separated by newlines. - Args: - filename (str): Filename - - Returns: - list: List of values loaded from file - """ - try: - data = [] - with open(filename, "r") as f: - while True: - line = f.readline() - if not line: - break - try: - data.append(json.loads(line)) - except EOFError: # reached end of file - break - except Exception as e: - _print_error(f"Error while loading data from file: {e}") - return None - - return data - - # - plotting + # -- plotting -- def _bokeh_plot_data_app(self, data, @@ -616,7 +691,7 @@ def update(step): doc.add_periodic_callback(update, plot_update_delay) return _app - def plot_data(self, x_var, y_vars, y_range=None, plot_update_delay=100, rollover=500): + def plot_data(self, x_var, y_vars, y_range=None, plot_update_delay=100, rollover=1000): """ Plots a bokeh figure with the streamed data. The plot is updated every plot_update_delay ms. The plot is interactive and can be zoomed in/out, panned, etc. The plot is shown in the notebook. Args: @@ -634,119 +709,65 @@ def plot_data(self, x_var, y_vars, y_range=None, plot_update_delay=100, rollover # check that x_var and y_vars are either streamed or monitored for _var in [x_var, *y_vars]: if not (_var in [var["name"] for var in self.watched_vars] or _var in [var["name"] for var in self.monitored_vars]): # FIXME - _print_error( + raise ValueError( f"PlottingError: {_var} is not being streamed or monitored.") - return # check buffer lengths are the same # wait until streaming buffers have been populated async def wait_for_streaming_buffers_to_arrive(): while not all(data['data'] for data in { var: _buffer for var, _buffer in self.last_streamed_buffer.items() if var in y_vars}.values()): - await asyncio.sleep(0.1) - asyncio.run(wait_for_streaming_buffers_to_arrive()) + await asyncio.sleep(0.01) + self.loop.run_until_complete( + wait_for_streaming_buffers_to_arrive()) if len(y_vars) > 1 and not all([len(self.last_streamed_buffer[y_var]) == len(self.last_streamed_buffer[y_vars[0]]) for y_var in y_vars[1:]]): - _print_error( + raise NotImplementedError( "PlottingError: plotting buffers of different length is not supported yet. Try using the same timestamp mode and type for your variables...") - return - - bokeh.io.output_notebook(INLINE) - bokeh.io.show(self._bokeh_plot_data_app(data={ - var: _buffer for var, _buffer in self.last_streamed_buffer.items() if var in y_vars}, x_var=x_var, - y_vars=y_vars, y_range=y_range, plot_update_delay=plot_update_delay, rollover=rollover)) - - # --- private methods --- # - - async def _process_data_msg(self, msg): - """ Process data message received from Bela. This function is called by the websocket listener when a data message is received. - - Args: - msg (bytestring): Data message received from Bela - """ - - global _channel, _type - try: - _, __ = _channel, _type - except NameError: # initialise global variables to None - _channel = None - _type = None - - # in case buffer is received whilst streaming mode is on but parsed after streaming_enabled has changed - _saving_enabled = copy.copy(self._saving_enabled) - if self._streaming_mode != "OFF": - if len(msg) in [3, 4]: # _channel can be either 1 or 2 bytes long - # parse buffer header - _channel, _type = re.search( - r'(\d+).*?(\w)', msg.decode()).groups() - _channel = int(_channel) - - assert _type in ['i', 'f', 'j', 'd', - 'c'], f"Unsupported type: {_type}" - assert _type == self._watcher_vars[_channel][ - 'type'], f"Type mismatch: {_type} != {self._watcher_vars[_channel]['type']}" + async def _async_plot_data(x_var, y_vars, y_range=None, plot_update_delay=100, rollover=1000): + bokeh.io.output_notebook(INLINE) + bokeh.io.show(self._bokeh_plot_data_app(data={ + var: _buffer for var, _buffer in self.last_streamed_buffer.items() if var in y_vars}, x_var=x_var, + y_vars=y_vars, y_range=y_range, plot_update_delay=plot_update_delay, rollover=rollover)) - # convert unsigned int to int -- struct does not support unsigned ints - _type = 'i' if _type == 'j' else _type + self.loop.run_until_complete(_async_plot_data( + x_var, y_vars, y_range, plot_update_delay, rollover)) - elif len(msg) > 3 and _channel is not None and _type is not None: - var_name = self._watcher_vars[_channel]['name'] - var_timestamp_mode = self._watcher_vars[_channel]["timestamp_mode"] +# -- utils -- - # parse buffer body - parsed_buffer = self._parse_binary_data( - msg, var_timestamp_mode, _type).copy() - - # put in processed_queue if callback is true - if self._on_buffer_callback_is_active or self._on_block_callback_is_active: - await self._processed_data_msg_queue.put({"name": var_name, "buffer": parsed_buffer}) - - # fixes bug where data is shifted by period - _var_streaming_buffers_queue = copy.copy( - self._streaming_buffers_queue[var_name]) - _var_streaming_buffers_queue.append(parsed_buffer) - self._streaming_buffers_queue[var_name] = _var_streaming_buffers_queue + def is_streaming(self): + """Returns True if the streamer is currently streaming, False otherwise. - # populate last streamed buffer - if self._mode == "STREAM": - self.last_streamed_buffer[var_name]["data"] = parsed_buffer["data"] - if var_timestamp_mode == "dense": - self.last_streamed_buffer[var_name]["timestamps"] = [ - parsed_buffer["ref_timestamp"] + i for i in range(0, len(parsed_buffer["data"]))] - elif var_timestamp_mode == "sparse": # sparse - self.last_streamed_buffer[var_name]["timestamps"] = [ - parsed_buffer["ref_timestamp"] + i for i in parsed_buffer["rel_timestamps"]] - elif self._mode == "MONITOR": - self.last_streamed_buffer[var_name] = { - "timestamp": parsed_buffer["timestamp"], "value": parsed_buffer["value"]} + Returns: + bool: Streaming status bool + """ + return True if self._streaming_mode != "OFF" else False - # save data to file if saving is enabled - if _saving_enabled: - _saving_var_filename = os.path.join(os.path.dirname( - self._saving_filename), f"{var_name}_{os.path.basename(self._saving_filename)}") - # save the data asynchronously - saving_task = self.loop.create_task( - self._save_data_to_file(_saving_var_filename, parsed_buffer)) - self._active_saving_tasks.append(saving_task) + def load_data_from_file(self, filename): + """ + Loads data from a file saved through the saving_enabled function in start_streaming() or stream_n_values(). The file should contain a list of dicts, each dict containing a variable name and a list of values. The list of dicts should be separated by newlines. + Args: + filename (str): Filename - # response to .peek() call - if self._mode == "MONITOR" and self._peek_response is not None: - # check that all the watched variables have been received - self._peek_response[self._watcher_vars[_channel]["name"]] = { - "timestamp": parsed_buffer["timestamp"], "value": parsed_buffer["value"]} - # notify peek() that data is available - if all(value is not None for value in self._peek_response.values()): - self._peek_response_available.set() + Returns: + list: List of values loaded from file + """ + try: + data = [] + with open(filename, "r") as f: + while True: + line = f.readline() + if not line: + break + try: + data.append(json.loads(line)) + except EOFError: # reached end of file + break + except Exception as e: + _print_error(f"Error while loading data from file: {e}") + return None - # if streaming buffers queue is full for watched variables and streaming mode is n_values - if self._streaming_mode == "N_VALUES": - _watched_vars = await self._async_watched_vars() - _monitored_vars = await self._async_monitored_vars() - obs_vars = _watched_vars if self._mode == "STREAM" else _monitored_vars - if all(len(self._streaming_buffers_queue[var["name"]]) == self._streaming_buffers_queue_length - for var in obs_vars): - self._streaming_mode = "OFF" - self._streaming_buffer_available.set() + return data async def _save_data_to_file(self, filename, msg): """ Saves data to file asynchronously. This function is called by _process_data_msg() when a buffer is received and saving is enabled. @@ -774,7 +795,7 @@ async def _save_data_to_file(self, filename, msg): # await self._async_remove_item_from_list(self._active_saving_tasks, asyncio.current_task()) def _generate_filename(self, saving_filename, saving_dir="./"): - """ Generates a filename for saving data by adding the variable name and a number at the end in case the filename already exists to avoid overwriting saved data. Pattern: varname_filename__idx.ext. This function is called by start_streaming() and stream_n_values() when saving is enabled. + """ Generates a filename for saving data by adding the variable name and a number at the end in case the filename already exists to avoid overwriting saved data. Pattern: varname_filename__idx.ext. This function is called by start_streaming() and stream_n_values() when saving is enabled. Args: saving_filename (str): Root filename @@ -836,14 +857,10 @@ def _check_periods(self, periods, variables): return periods async def _async_disconnect(self): + """ Cancels existing tasks + """ await super()._async_disconnect() if self._on_buffer_callback_worker_task is not None and not self._on_buffer_callback_worker_task.done(): self._on_buffer_callback_worker_task.cancel() if self._on_block_callback_worker_task is not None and not self._on_block_callback_worker_task.done(): self._on_block_callback_worker_task.cancel() - - def disconnect(self): - self.loop.run_until_complete(self._async_disconnect()) - - # def __del__(self): - # super().__del__() diff --git a/pybela/Watcher.py b/pybela/Watcher.py index 5cd5d8d..6525547 100644 --- a/pybela/Watcher.py +++ b/pybela/Watcher.py @@ -1,11 +1,9 @@ import asyncio -import nest_asyncio import websockets import json import errno import struct import os -import gc from .utils import _print_error, _print_warning, _print_ok @@ -32,11 +30,14 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add self.ws_data_add = f"ws://{self.ip}:{self.port}/{self.data_add}" self.ws_ctrl = None self.ws_data = None + self._watcher_vars = None + self._mode = "WATCH" + global _pybela_ws_register try: _ = _pybela_ws_register except NameError: # initialise _pybela_ws_register only once in runtime - _pybela_ws_register = {"loop": None, + _pybela_ws_register = {"event-loop": None, "WATCH": {}, "STREAM": {}, "LOG": {}, @@ -46,23 +47,13 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add self._pybela_ws_register = _pybela_ws_register # background event loop - # If no loop exists, create a new one - if self._pybela_ws_register["loop"] is None: + if self._pybela_ws_register["event-loop"] is None: self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) - self._pybela_ws_register["loop"] = self.loop - else: - self.loop = self._pybela_ws_register["loop"] - # self.loop.run_forever() - # self.loop = asyncio.new_event_loop() - # asyncio.set_event_loop(self.loop) - # self.loop.run_forever() - - # self.loop_thread = threading.Thread( - # target=self._run_event_loop, daemon=True) - # self.loop_thread.start() - # self.loop = asyncio.get_event_loop() + self._pybela_ws_register["event-loop"] = self.loop + else: # if loop exists, use the existing one + self.loop = self._pybela_ws_register["event-loop"] # tasks self._ctrl_listener_task = None @@ -77,19 +68,9 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add self._to_send_data_msg_queue = asyncio.Queue(loop=self.loop) self._to_send_ctrl_msg_queue = asyncio.Queue(loop=self.loop) - self._watcher_vars = None - - self._mode = "WATCH" - # debug self._printall_responses = False - # event loop needs to be nested - otherwise it conflicts with jupyter's event loop - # nest_asyncio.apply() - def _run_event_loop(self): - asyncio.set_event_loop(self.loop) - self.loop.run_forever() - # properties @property @@ -98,7 +79,7 @@ def sample_rate(self): @property def watcher_vars(self): - """Returns variables in watcher with their properties (name, type, timestamp_mode, log_filename, data_length) + """Returns variables in watcher with their properties (name, type, timestamp_mode, log_filename, data_length). Can't be used in async functions, use _async_watcher_vars instead. Returns: list of dicts: List of variables in watcher and their properties @@ -110,6 +91,11 @@ def watcher_vars(self): return self._watcher_vars # updates every time start is called async def _async_watcher_vars(self): + """Asynchronous version of watcher_vars + + Returns: + list of dicts: List of variables in watcher and their properties + """ if self._watcher_vars == None: _list = await self._async_list() self._watcher_vars = self._filtered_watcher_vars( @@ -118,7 +104,7 @@ async def _async_watcher_vars(self): @property def watched_vars(self): - """Returns a list of the variables in the watcher that are being watched (i.e., whose data is being sent over websockets for streaming or monitoring) + """Returns a list of the variables in the watcher that are being watched (i.e., whose data is being sent over websockets for streaming or monitoring). Can't be used in async functions, use _async_watched_vars instead. Returns: list of str: List of watched variables @@ -127,12 +113,17 @@ def watched_vars(self): return self._filtered_watcher_vars(_list["watchers"], lambda var: var["watched"]) async def _async_watched_vars(self): + """Async version of watched_vars + + Returns: + list of str: List of watched variables + """ _list = await self._async_list() return self._filtered_watcher_vars(_list["watchers"], lambda var: var["watched"]) @property def unwatched_vars(self): - """Returns a list of the variables in the watcher that are not being watched (i.e., whose data is NOT being sent over websockets for streaming or monitoring) + """Returns a list of the variables in the watcher that are not being watched (i.e., whose data is NOT being sent over websockets for streaming or monitoring). Can't be used in async functions, use _async_unwatched_vars instead. Returns: list of str: List of unwatched variables @@ -141,10 +132,15 @@ def unwatched_vars(self): return self._filtered_watcher_vars(_list["watchers"], lambda var: not var["watched"]) async def _async_unwatched_vars(self): + """Async version of unwatched_vars + + Returns: + list of str: List of unwatched variables + """ _list = await self._async_list() return self._filtered_watcher_vars(_list["watchers"], lambda var: not var["watched"]) - # --- public methods --- # + # --- connection methods --- # def connect(self): """Attempts to establish a WebSocket connection and prints a message indicating success or failure. @@ -198,15 +194,15 @@ async def _async_connect(): # Connect to the data websocket self.ws_data = await websockets.connect(self.ws_data_add) - # start data processing and sending tasks - self._process_received_data_msg_task = self.loop.create_task( - self._process_data_msg_worker()) - self._send_data_msg_task = self.loop.create_task( - self._send_data_msg_worker()) + # start data sending and processing tasks self._send_ctrl_msg_task = self.loop.create_task( self._send_ctrl_msg_worker()) + self._send_data_msg_task = self.loop.create_task( + self._send_data_msg_worker()) + self._process_received_data_msg_task = self.loop.create_task( + self._process_data_msg_worker()) - # Start listener tasks + # start listener tasks self._ctrl_listener_task = self.loop.create_task(self._async_start_listener( self.ws_ctrl, self.ws_ctrl_add)) self._data_listener_task = self.loop.create_task(self._async_start_listener( @@ -228,36 +224,104 @@ async def _async_connect(): return self.loop.run_until_complete(_async_connect()) def is_connected(self): + """Check if the websocket is connected + Returns: + bool: True if connected, False otherwise + """ + return True if (self.ws_ctrl is not None and self.ws_ctrl.state == 1) and (self.ws_data is not None and self.ws_data.state == 1) else False async def _async_disconnect(self): + """Disconnects the websockets. Closes the websockets and cancels the keepalive task.""" # close websockets - wss = [self.ws_ctrl, self.ws_data] - for ws in wss: + for ws in [self.ws_ctrl, self.ws_data]: if ws is not None and ws.state == 1: await ws.close() ws.keepalive_task.cancel() # cancel keepalive task def disconnect(self): - """Closes websockets + """Closes websockets. Sync wrapper for _async_disconnect. """ self.loop.run_until_complete(self._async_disconnect()) - pass + async def _async_cancel_tasks(self, tasks): + """Cancels tasks + """ + cancel_tasks = [] + for task in tasks: + if task is not None and not task.done(): + task.cancel() + cancel_tasks.append(task) + await asyncio.gather(*cancel_tasks, return_exceptions=True) - async def _async_list(self): - await self._async_send_ctrl_msg({"watcher": [{"cmd": "list"}]}) - # Wait for the list response to be available - list_res = await self._list_response_queue.get() - self._list_response_queue.task_done() - return list_res + async def _async_cleanup(self): + """Cleans up tasks + """ + tasks = [self._ctrl_listener_task, + self._data_listener_task, + self._process_received_data_msg_task, + self._send_data_msg_task, + self._send_ctrl_msg_task + ] + await self._async_cancel_tasks(tasks) + await self._async_disconnect() - def list(self): - """ Asks the watcher for the list of variables and their properties and returns it + def cleanup(self): + """Cleans up tasks. Synchronous wrapper for _async_cleanup """ - return self.loop.run_until_complete(self._async_list()) + self.loop.run_until_complete(self._async_cleanup()) + + # --- message sending methods --- # + + async def _send_data_msg_worker(self): + """ Send data message to websocket. Runs as long as websocket is open. + """ + while self.ws_data is not None and self.ws_data.state == 1: + msg = await self._to_send_data_msg_queue.get() + await self.ws_data.send(msg) + self._to_send_data_msg_queue.task_done() + + async def _send_ctrl_msg_worker(self): + """ Send control message to websocket. Runs as long as websocket is open. + """ + while self.ws_ctrl is not None and self.ws_ctrl.state == 1: + msg = await self._to_send_ctrl_msg_queue.get() + msg = json.dumps(msg) + await self.ws_ctrl.send(msg) + self._to_send_ctrl_msg_queue.task_done() + + async def _async_send_msg(self, ws_address, msg): + """Send message to websocket + + Args: + ws_address (str): Websocket address + msg (str): Message to send + """ + try: + if ws_address == self.ws_data_add and self.ws_data is not None and self.ws_data.state == 1: + self._to_send_data_msg_queue.put_nowait(msg) + elif ws_address == self.ws_ctrl_add and self.ws_ctrl is not None and self.ws_ctrl.state == 1: + # msg = json.dumps(msg) + self._to_send_ctrl_msg_queue.put_nowait(msg) + except Exception as e: + _handle_connection_exception(ws_address, e, "sending message") + return 0 + + def _send_msg(self, ws_address, msg): + """Send message to websocket. Sync wrapper for _async_send_msg. Can be used in synchronous functions. + + Args: + ws_address (str): Websocket address + msg (str): Message to send + """ + return self.loop.create_task(self._async_send_msg(ws_address, msg)) async def _async_send_ctrl_msg(self, msg): + """Send control message. Async version of send_ctrl_msg. + + Args: + msg (str): Message to send to the Bela watcher. Example: {"watcher": [{"cmd": "list"}]} + """ await self._async_send_msg(self.ws_ctrl_add, msg) def send_ctrl_msg(self, msg): @@ -268,9 +332,26 @@ def send_ctrl_msg(self, msg): """ self._send_msg(self.ws_ctrl_add, msg) - # --- private methods --- # + ##  -- list -- ## + + async def _async_list(self): + """ Asks the watcher for the list of variables and their properties and returns it. + + Returns: + dict: Dictionary with the list of variables and their properties + """ + self.send_ctrl_msg({"watcher": [{"cmd": "list"}]}) + # Wait for the list response to be available + list_res = await self._list_response_queue.get() + self._list_response_queue.task_done() + return list_res + + def list(self): + """ Sync wrapper for _async_list + """ + return self.loop.run_until_complete(self._async_list()) - # start listener + # -- listener methods -- # async def _async_start_listener(self, ws, ws_address): """Start listener for websocket @@ -290,56 +371,17 @@ async def _async_start_listener(self, ws, ws_address): _msg = json.loads(msg) # response to list cmd if "watcher" in _msg.keys() and "sampleRate" in _msg["watcher"].keys(): - self._list_response_queue.put_nowait(_msg["watcher"]) + self._list_response_queue.put_nowait( + _msg["watcher"]) else: print(msg) - except Exception or asyncio.exceptions.CancelledError as e: + except Exception as e: if ws.state == 1: # otherwise websocket was closed intentionally _handle_connection_exception( ws_address, e, "receiving message") - # send message - async def _async_send_msg(self, ws_address, msg): - try: - if ws_address == self.ws_data_add and self.ws_data is not None and self.ws_data.state == 1: - await self._to_send_data_msg_queue.put(msg) - elif ws_address == self.ws_ctrl_add and self.ws_ctrl is not None and self.ws_ctrl.state == 1: - # msg = json.dumps(msg) - await self._to_send_ctrl_msg_queue.put(msg) - except Exception as e: - _handle_connection_exception(ws_address, e, "sending message") - return 0 - - def _send_msg(self, ws_address, msg): - """Send message to websocket - - Args: - ws_address (str): Websocket address - msg (str): Message to send - """ - return self.loop.create_task(self._async_send_msg(ws_address, msg)) - - # send messages - - async def _send_data_msg_worker(self): - """ Send data message to websocket. Runs as long as websocket is open. - """ - while self.ws_data is not None and self.ws_data.state == 1: - msg = await self._to_send_data_msg_queue.get() - await self.ws_data.send(msg) - self._to_send_data_msg_queue.task_done() - - async def _send_ctrl_msg_worker(self): - """ Send control message to websocket. Runs as long as websocket is open. - """ - while self.ws_ctrl is not None and self.ws_ctrl.state == 1: - msg = await self._to_send_ctrl_msg_queue.get() - msg = json.dumps(msg) - await self.ws_ctrl.send(msg) - self._to_send_ctrl_msg_queue.task_done() - - # process messages + # -- data processing methods -- # async def _process_data_msg_worker(self): """Process data message. @@ -359,8 +401,6 @@ async def _process_data_msg(self, msg): Args: msg (str): Bytestring with data """ - _msg = json.loads(msg) - print(_msg) pass def _process_ctrl_msg(self, msg): @@ -397,11 +437,17 @@ def _parse_binary_data(self, binary_data, timestamp_mode, _type): # sparse mode if timestamp_mode == "sparse": # ensure that the buffer is the correct size (remove padding) + binary_data = binary_data[:struct.calcsize("Q") + data_length*struct.calcsize( _type)+data_length*struct.calcsize("I")] - ref_timestamp, *_buffer = struct.unpack('Q' + f"{_type}" * data_length - + 'I'*data_length, binary_data) + try: + ref_timestamp, *_buffer = struct.unpack('Q' + f"{_type}" * data_length + + 'I'*data_length, binary_data) + except struct.error as e: + _print_error( + f"Error parsing buffer: {e}. Received buffer of length: {len(binary_data)}") + return None data = _buffer[:data_length] # remove padding rel_timestamps = _buffer[data_length:][:data_length] @@ -434,13 +480,16 @@ def _parse_binary_data(self, binary_data, timestamp_mode, _type): # --- utils --- # def wait(self, time_in_seconds): + """Wait for a given amount of time. Can't be used in async functions.""" self.loop.run_until_complete(asyncio.sleep(time_in_seconds)) async def _async_get_latest_timestamp(self): + """Get latest timestamp. Async version of get_latest_timestamp.""" _list = await self._async_list() return _list["timestamp"] def get_latest_timestamp(self): + """Get latest timestamp. Can't be used in async functions""" return self.list()["timestamp"] async def _async_remove_item_from_list(self, _list, task): @@ -495,7 +544,11 @@ def _var_arg_checker(self, variables): return variables def _generate_local_filename(self, local_path): - # if file already exists, throw a warning and add number at the end of the filename + """Generate local filename. If the file already exists, add a number at the end of the filename. + + Args: + local_path (str): Path to the file + """ new_local_path = local_path # default if os.path.exists(local_path): base, ext = os.path.splitext(local_path) @@ -596,44 +649,10 @@ def get_buffer_size(self, var_type, timestamp_mode): # return error message return 0 - async def _async_cancel_tasks(self, tasks): - """Cancels tasks - """ - cancel_tasks = [] - for task in tasks: - if task is not None and not task.done(): - task.cancel() - cancel_tasks.append(task) - await asyncio.gather(*cancel_tasks, return_exceptions=True) - - async def _async_cleanup(self): - """Cleans up tasks - """ - tasks = [self._ctrl_listener_task, - self._data_listener_task, - self._process_received_data_msg_task, - self._send_data_msg_task, - self._send_ctrl_msg_task - ] - await self._async_cancel_tasks(tasks) - await self._async_disconnect() - - def cleanup(self): - """Cleans up tasks. Synchronous wrapper for _async_cleanup - """ - self.loop.run_until_complete(self._async_cleanup()) - # destructor def __del__(self): - pass - # self.disconnect() # stop websockets and cancel tasks - # stop event loop - # self.wait(0.5) - # if self.loop.is_running(): - # self.loop.stop() - # self.loop.close() - # self.loop_thread.join() + pass # __del__ can't run asynchronous code, so cleanup() should be called manually def _handle_connection_exception(ws_address, exception, action): diff --git a/test/test.py b/test/test.py index 32302a3..e03271d 100644 --- a/test/test.py +++ b/test/test.py @@ -42,7 +42,6 @@ def setUp(self): "myvar4" # sparse double ] self.saving_dir = "./test" - self.saving_filename = "test_streamer_save.txt" def tearDown(self): @@ -90,7 +89,7 @@ def __test_buffers(self, mode): "The ref_timestamp and the first item of data buffer should be the same") self.assertEqual(_buffer["ref_timestamp"]+var["data_length"]-1, _buffer["data"][-1], "The last data item should be equal to the ref_timestamp plus the length of the buffer") # this test will fail if the Bela program has been streaming for too long and there are truncating errors. If this test fails, try stopping and rerunning hte Bela program again - + # delete files for var in self.streaming_vars: remove_file(os.path.join(self.saving_dir, f"{var}_{self.saving_filename}")) @@ -260,10 +259,8 @@ def test_logged_files_wo_transfer(self): local_paths = {} for var in file_paths["remote_paths"]: filename = os.path.basename(file_paths["remote_paths"][var]) - local_paths[var] = self.logger._generate_local_filename( - os.path.join(self.logging_dir, filename)) - self.logger.copy_file_from_bela(remote_path=file_paths["remote_paths"][var], - local_path=local_paths[var]) + local_paths[var] = self.logger.copy_file_from_bela(remote_path=file_paths["remote_paths"][var], + local_path=filename) # test logged data self._test_logged_data(self.logger, self.logging_vars, local_paths) @@ -435,7 +432,7 @@ def remove_file(file_path): # unittest.main(verbosity=2) # select which tests to run - n = 1 + n = 2 for i in range(n): print(f"\n\n....Running test {i+1}/{n}") From af1ef919df63250fe5c294bd0d13a055d0039d38 Mon Sep 17 00:00:00 2001 From: Teresa Pelinski Date: Mon, 16 Dec 2024 21:16:09 +0000 Subject: [PATCH 4/5] updated notebooks (removed asyncio sleep), added auto nest_asyncio if in notebook, minor fixes --- pybela/Monitor.py | 2 +- pybela/Watcher.py | 8 + .../1_Streamer-Bela-to-python-basics.ipynb | 733 +++++++++--------- .../2_Streamer-Bela-to-python-advanced.ipynb | 413 +++++----- .../notebooks/3_Streamer-python-to-Bela.ipynb | 479 ++++++------ tutorials/notebooks/4_Monitor.ipynb | 19 +- tutorials/notebooks/5_Logger.ipynb | 9 +- tutorials/notebooks/6_Controller.ipynb | 4 +- .../notebooks/7_Sparse-timestamping.ipynb | 241 +++++- 9 files changed, 1073 insertions(+), 835 deletions(-) diff --git a/pybela/Monitor.py b/pybela/Monitor.py index 750a437..5fc9c82 100644 --- a/pybela/Monitor.py +++ b/pybela/Monitor.py @@ -66,7 +66,7 @@ async def _async_peek(variables): # res = self.list() # return {var: next(r["value"] for r in res if r["name"] == var) for var in variables} - def start_monitoring(self, variables=[], periods=[], saving_enabled=False, saving_filename="monitor.txt", saving_dir="/."): + def start_monitoring(self, variables=[], periods=[], saving_enabled=False, saving_filename="monitor.txt", saving_dir="./"): """ Starts the monitoring session. The session can be stopped with stop_monitoring(). diff --git a/pybela/Watcher.py b/pybela/Watcher.py index 6525547..6c05111 100644 --- a/pybela/Watcher.py +++ b/pybela/Watcher.py @@ -4,6 +4,7 @@ import errno import struct import os +import nest_asyncio from .utils import _print_error, _print_warning, _print_ok @@ -46,6 +47,13 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add self._pybela_ws_register = _pybela_ws_register + # if running in jupyter notebook, enable nest_asyncio + try: + get_ipython().__class__.__name__ + nest_asyncio.apply() + except NameError: + pass + # background event loop # If no loop exists, create a new one if self._pybela_ws_register["event-loop"] is None: diff --git a/tutorials/notebooks/1_Streamer-Bela-to-python-basics.ipynb b/tutorials/notebooks/1_Streamer-Bela-to-python-basics.ipynb index 09ac92c..9d0985a 100644 --- a/tutorials/notebooks/1_Streamer-Bela-to-python-basics.ipynb +++ b/tutorials/notebooks/1_Streamer-Bela-to-python-basics.ipynb @@ -1,370 +1,369 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# pybela Tutorial 1: Streamer – Bela to python basics\n", - "This notebook is a tutorial for the Streamer class in the pybela python library. You can use the Streamer to stream data from Bela to python or vice versa. \n", - "\n", - "In this tutorial we will be looking at sending data from Bela to python. The Streamer allows you to start and stop streaming, to stream a given number of data points, to plot the data as it arrives, and to save and load the streamed data into `.txt` files. \n", - "\n", - "The complete documentation for the pybela library can be found in [https://belaplatform.github.io/pybela/](https://belaplatform.github.io/pybela/).\n", - "\n", - "To run this tutorial, first copy the `bela-code/potentiometers` project onto Bela. If your Bela is connected to your laptop, you can run the cell below:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!rsync -rvL ../bela-code/potentiometers root@bela.local:Bela/projects" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Then you can compile and run the project using either the IDE or by running the following command in the Terminal:\n", - "```bash\n", - "ssh root@bela.local \"make -C Bela stop Bela PROJECT=potentiometers run\" \n", - "```\n", - "(Running this on a jupyter notebook will block the cell until the program is stopped on Bela.)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Setting up the circuit\n", - "In this example we will be using two potentiometers as our analog signals, but you can connect whichever sensors you like to analog channels 0 and 1.\n", - "\n", - "Potentiometers have 3 pins. To connect a potentiometer to Bela, attach the left pin to the Bela 3.3V pin, the central pin to the desired analog input (e.g. 0) and the right pin to the Bela GND pin:\n", - "\n", - "

\n", - "\n", - "

\n", - "\n", - "### Taking a look at the Bela C++ code\n", - "If you take a look into the Bela code (in `bela-code/potentiometers/render.cpp`), you will see that the variables `pot1` and `pot2` are defined in a particular way:\n", - "\n", - "```cpp\n", - "Watcher pot1(\"pot1\");\n", - "Watcher pot2(\"pot2\");\n", - "```\n", - "\n", - "This means that the variables `pot1` and `pot2` are being \"watched\" and hence we can request their values to be streamed to this notebook using the pybela Streamer class. The watcher will stream a buffer containing timestamp and variable value information. Take a look at the `render` loop:\n", - "\n", - "```cpp\n", - "void render(BelaContext *context, void *userData)\n", - "{\n", - "\tfor(unsigned int n = 0; n < context->audioFrames; n++) {\n", - "\t\tif(gAudioFramesPerAnalogFrame && !(n % gAudioFramesPerAnalogFrame)) {\n", - "\t\t\t\n", - "\t\t\tuint64_t frames = context->audioFramesElapsed/gAudioFramesPerAnalogFrame + n/gAudioFramesPerAnalogFrame;\n", - "\t\t\tBela_getDefaultWatcherManager()->tick(frames); // watcher timestamps\n", - "\t\t\t\n", - "\t\t\tpot1 = analogRead(context, n/gAudioFramesPerAnalogFrame, gPot1Ch);\n", - "\t\t\tpot2 = analogRead(context, n/gAudioFramesPerAnalogFrame, gPot2Ch);\n", - "\t\t\t\n", - "\t\t}\n", - "\t}\n", - "}\n", - "```\n", - "\n", - "we are reading the values of the potentiometer (with `analogRead()`) at every audio frame, and assigning them to their corresponding variable (`pot1` and `pot2`). In order for the Bela Watcher to know at which timestamp this happens, we need to \"tick\" the Watcher clock, we do this in line 30 with:\n", - "```cpp\n", - "\t\t\tBela_getDefaultWatcherManager()->tick(frames); // watcher timestamps\n", - "```\n", - "\n", - "If you want to take a look at more advanced ways of watching variables, take a look at the Logger notebook. But enough with C++, let's take a look at the pybela Streamer class and its usage. \n", - "\n", - "### Getting started\n", - "Once you have the circuit set up, build and run the Bela project `potentiometers`. Once running, we are ready to interact with it form this notebook. We'll start by importing some necessary libraries and setting the `BOKEH_ALLOW_WS_ORIGIN` environment that will allow us to visualise the bokeh plots (comment/uncomment depending on if you are running this notebook from a jupyter notebook or VSCode)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import asyncio\n", - "import pandas as pd\n", - "from pybela import Streamer\n", - "import os\n", - "# os.environ['BOKEH_ALLOW_WS_ORIGIN'] = \"1t4j54lsdj67h02ol8hionopt4k7b7ngd9483l5q5pagr3j2droq\" # uncomment if running on vscode\n", - "os.environ['BOKEH_ALLOW_WS_ORIGIN'] = \"localhost:8888\" # uncomment if running on jupyter" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now let's initialise the streamer and connect it to the Bela websocket. If the connection fails, make sure Bela is connected to your laptop and that the `potentiometer` project is running on Bela." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "streamer = Streamer()\n", - "streamer.connect()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's start by streaming the values of potentiometer 1 and 2. For that, we call `streamer.start_streaming(variables=[\"pot1\", \"pot2\"])`. This will request the values of the variables `pot1` and `pot`. We can visualise those values as they arrive by plotting them using `streamer.plot_data(x_var=\"pot1\", y_vars=[\"pot1\", \"pot2\"], y_range=[0,1])`. The argument `x_var` determines which variable will provide the timestamps for the x axis, and the argument `y_vars` expects a list of variables that are currently being streamed or monitored. `y_range` determines the range of the y-axis." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "streamer.start_streaming(variables=[\"pot1\", \"pot2\"])\n", - "streamer.plot_data(x_var=\"pot1\", y_vars=[\"pot1\", \"pot2\"], y_range=[0, 1], rollover=10000)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can stop streaming the values of potentiometer 1 and 2 by calling `streamer.stop_streaming(variables=[\"pot1\", \"pot2\"])`. You can also call `streamer.stop_streaming()` which will stop streaming all the available variables in the watcher (in this case, both `pot1` and `pot2`)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "streamer.stop_streaming()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Using asyncio to stream data for a fixed amount of time\n", - "You can use the `asyncio` python library to stream data for a fixed amount of time. Note: you need to use `asyncio.sleep` instead of `time.sleep`, since the latter pauses the entire program (including the streaming tasks running in the background)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "streamer.start_streaming(variables=[\"pot2\"])\n", - "streamer.plot_data(x_var=\"pot2\", y_vars=[\"pot2\"], y_range=[0, 1])\n", - "await asyncio.sleep(5)\n", - "streamer.stop_streaming()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Scheduling streaming sessions\n", - "You can schedule a streaming session to start and stop at a specific time using the `schedule_streaming()` method. This method takes the same arguments as `start_streaming()`, but it also takes a `timestamps` and `durations` argument." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "latest_timestamp = streamer.get_latest_timestamp() # get the latest timestamp\n", - "sample_rate = streamer.sample_rate # get the sample rate\n", - "start_timestamp = latest_timestamp + sample_rate # start streaming 1 second after the latest timestamp\n", - "duration = sample_rate # stream for 2 seconds\n", - "\n", - "streamer.schedule_streaming(\n", - " variables=[\"pot1\", \"pot2\"],\n", - " timestamps=[start_timestamp, start_timestamp],\n", - " durations=[duration, duration],\n", - " saving_enabled=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Note on streaming variables assigned at low frequency rates\n", - "The data buffers sent from Bela have fixed sizes. The buffers will only be sent when they are full, unless you use the streaming with scheduling feature (explained below). If the variables you are streaming are assigned at too low rates, these buffers will take too long to fill up and the data will be either sent to python with a delay or not sent at all (if the buffer is never filled). For example, floats using dense timestamping are sent in buffers of 1024 values. If the float variable is assigned once every 12 milliseconds, filling a buffer will take 1024/(1/0.012) = 12.3 seconds. \n", - "Hence, the streaming mode is not ideal for variables assigned at low rates, but rather for variables that are assigned quite frequently (e.g. at audio rate). If you want to stream variables that are assigned at lower rates, you can use the streaming with scheduling feature, or monitor or log the variable instead." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Retrieving the data\n", - "You can access the data streamed in `streamer.streaming_buffers_data`. We can use the pandas data manipulation library for printing the data onto a table:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df = pd.DataFrame(streamer.streaming_buffers_data[\"pot2\"])\n", - "df.head() # head shows only the first 5 rows" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As you can see, `streaming_buffers_data` only retrieves the variable values but not its timestamps. If you want to retrieve the timestamps, you can access `streaming_buffers_queue[\"pot2\"]`. This will return a list in which every item is a timestamped buffer:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "streamer.streaming_buffers_queue[\"pot2\"][0]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the buffer `ref_timestamp` corresponds to the timestamp of the first value of the buffer (`streaming_buffers_queue[\"pot2\"][0][\"data\"][0]`). If the Bela Watcher is ticked once per analog frame (as it is the case in the `potentiometer` code) and the variable `pot2` is assigned also once per analog frame, the timestamps of the rest of the values in the data buffer correspond to the increasing timestamps:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "data_timestamps = []\n", - "data_values = []\n", - "\n", - "def flatten_buffers_queue(_buffer_queue):\n", - " for _buffer in _buffer_queue:\n", - " ref_timestamp = _buffer[\"ref_timestamp\"]\n", - " data_timestamps.extend([ref_timestamp + i for i in range(len(_buffer[\"data\"]))])\n", - " data_values.extend(_buffer[\"data\"])\n", - " \n", - " return data_timestamps, data_values\n", - "\n", - "data_timestamps, data_values = flatten_buffers_queue(streamer.streaming_buffers_queue[\"pot2\"])\n", - " \n", - "df = pd.DataFrame({\"timestamp\": data_timestamps, \"value\": data_values})\n", - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "More advanced timestamping methods will be shown in the tutorial notebook `7_Sparse_timestamping.ipynb`\n", - "\n", - "There is a limited amount of data that is stored in the streamer. This quantity can be modified by changing the buffer queue length. The streamer receives the data in buffers of fixed length that get stored in a queue that also has a fixed length. You can calculate the maximum amount of data the streamer can store for each variable:\n", - "\n", - "note: `streamer.watcher_vars` returns information of the variables available in the watcher, that is, variables that have been defined within the Watcher class in the Bela code and that are available for streaming, monitoring or logging." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(f\"Buffer queue length: {streamer.streaming_buffers_queue_length}\")\n", - "\n", - "for var in streamer.watcher_vars: \n", - " print(f'Variable: {var[\"name\"]}, buffer length: {var[\"data_length\"]}, max data stored in streamer: {var[\"data_length\"]*streamer.streaming_buffers_queue_length}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also modify the queue length:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "streamer.streaming_buffers_queue_length = 10" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Saving the streamed data\n", - "Every time you start a new streaming session (e.g. you call `start_streaming()` or `stream_n_values()`), the data stored in the streamer from the previous streaming session will be deleted. If you want to store the streamed data, you can do so by setting `saving_enabled=True` when calling `start_streaming()` or `stream_n_values()`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "streamer.start_streaming(variables=[var[\"name\"] for var in streamer.watcher_vars], saving_enabled=True, saving_filename=\"test.txt\")\n", - "await asyncio.sleep(3)\n", - "streamer.stop_streaming()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can load the data stored using the `load_data_from_file` method. This will return the buffers queue. Again, we can flatten it using the `flatten_buffers_queue()` function we defined above:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "data_timestamps, data_values = flatten_buffers_queue(streamer.load_data_from_file(\"pot1_test.txt\"))\n", - "\n", - "df=pd.DataFrame({\"timestamp\": data_timestamps, \"value\": data_values})\n", - "df.head()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# pybela Tutorial 1: Streamer – Bela to python basics\n", + "This notebook is a tutorial for the Streamer class in the pybela python library. You can use the Streamer to stream data from Bela to python or vice versa. \n", + "\n", + "In this tutorial we will be looking at sending data from Bela to python. The Streamer allows you to start and stop streaming, to stream a given number of data points, to plot the data as it arrives, and to save and load the streamed data into `.txt` files. \n", + "\n", + "The complete documentation for the pybela library can be found in [https://belaplatform.github.io/pybela/](https://belaplatform.github.io/pybela/).\n", + "\n", + "To run this tutorial, first copy the `bela-code/potentiometers` project onto Bela. If your Bela is connected to your laptop, you can run the cell below:" + ] }, - "nbformat": 4, - "nbformat_minor": 4 + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!rsync -rvL ../bela-code/potentiometers root@bela.local:Bela/projects" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then you can compile and run the project using either the IDE or by running the following command in the Terminal:\n", + "```bash\n", + "ssh root@bela.local \"make -C Bela stop Bela PROJECT=potentiometers run\" \n", + "```\n", + "(Running this on a jupyter notebook will block the cell until the program is stopped on Bela.)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Setting up the circuit\n", + "In this example we will be using two potentiometers as our analog signals, but you can connect whichever sensors you like to analog channels 0 and 1.\n", + "\n", + "Potentiometers have 3 pins. To connect a potentiometer to Bela, attach the left pin to the Bela 3.3V pin, the central pin to the desired analog input (e.g. 0) and the right pin to the Bela GND pin:\n", + "\n", + "

\n", + "\n", + "

\n", + "\n", + "### Taking a look at the Bela C++ code\n", + "If you take a look into the Bela code (in `bela-code/potentiometers/render.cpp`), you will see that the variables `pot1` and `pot2` are defined in a particular way:\n", + "\n", + "```cpp\n", + "Watcher pot1(\"pot1\");\n", + "Watcher pot2(\"pot2\");\n", + "```\n", + "\n", + "This means that the variables `pot1` and `pot2` are being \"watched\" and hence we can request their values to be streamed to this notebook using the pybela Streamer class. The watcher will stream a buffer containing timestamp and variable value information. Take a look at the `render` loop:\n", + "\n", + "```cpp\n", + "void render(BelaContext *context, void *userData)\n", + "{\n", + "\tfor(unsigned int n = 0; n < context->audioFrames; n++) {\n", + "\t\tif(gAudioFramesPerAnalogFrame && !(n % gAudioFramesPerAnalogFrame)) {\n", + "\t\t\t\n", + "\t\t\tuint64_t frames = context->audioFramesElapsed/gAudioFramesPerAnalogFrame + n/gAudioFramesPerAnalogFrame;\n", + "\t\t\tBela_getDefaultWatcherManager()->tick(frames); // watcher timestamps\n", + "\t\t\t\n", + "\t\t\tpot1 = analogRead(context, n/gAudioFramesPerAnalogFrame, gPot1Ch);\n", + "\t\t\tpot2 = analogRead(context, n/gAudioFramesPerAnalogFrame, gPot2Ch);\n", + "\t\t\t\n", + "\t\t}\n", + "\t}\n", + "}\n", + "```\n", + "\n", + "we are reading the values of the potentiometer (with `analogRead()`) at every audio frame, and assigning them to their corresponding variable (`pot1` and `pot2`). In order for the Bela Watcher to know at which timestamp this happens, we need to \"tick\" the Watcher clock, we do this in line 30 with:\n", + "```cpp\n", + "\t\t\tBela_getDefaultWatcherManager()->tick(frames); // watcher timestamps\n", + "```\n", + "\n", + "If you want to take a look at more advanced ways of watching variables, take a look at the Logger notebook. But enough with C++, let's take a look at the pybela Streamer class and its usage. \n", + "\n", + "### Getting started\n", + "Once you have the circuit set up, build and run the Bela project `potentiometers`. Once running, we are ready to interact with it form this notebook. We'll start by importing some necessary libraries and setting the `BOKEH_ALLOW_WS_ORIGIN` environment that will allow us to visualise the bokeh plots (comment/uncomment depending on if you are running this notebook from a jupyter notebook or VSCode)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from pybela import Streamer\n", + "import os\n", + "os.environ['BOKEH_ALLOW_WS_ORIGIN'] = \"1t4j54lsdj67h02ol8hionopt4k7b7ngd9483l5q5pagr3j2droq\" # uncomment if running on vscode\n", + "# os.environ['BOKEH_ALLOW_WS_ORIGIN'] = \"localhost:8888\" # uncomment if running on jupyter" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's initialise the streamer and connect it to the Bela websocket. If the connection fails, make sure Bela is connected to your laptop and that the `potentiometer` project is running on Bela." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer = Streamer()\n", + "streamer.connect()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's start by streaming the values of potentiometer 1 and 2. For that, we call `streamer.start_streaming(variables=[\"pot1\", \"pot2\"])`. This will request the values of the variables `pot1` and `pot`. We can visualise those values as they arrive by plotting them using `streamer.plot_data(x_var=\"pot1\", y_vars=[\"pot1\", \"pot2\"], y_range=[0,1])`. The argument `x_var` determines which variable will provide the timestamps for the x axis, and the argument `y_vars` expects a list of variables that are currently being streamed or monitored. `y_range` determines the range of the y-axis." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer.start_streaming(variables=[\"pot1\", \"pot2\"])\n", + "streamer.plot_data(x_var=\"pot1\", y_vars=[\"pot1\", \"pot2\"], y_range=[0, 1], rollover=10000)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can stop streaming the values of potentiometer 1 and 2 by calling `streamer.stop_streaming(variables=[\"pot1\", \"pot2\"])`. You can also call `streamer.stop_streaming()` which will stop streaming all the available variables in the watcher (in this case, both `pot1` and `pot2`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer.stop_streaming()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Using `.wait` to stream data for a fixed amount of time\n", + "You can use the `.wait` method to stream data for a fixed amount of time. Note: you need to use `.wait` method instead of `time.sleep`, since the latter pauses the entire program (including the streaming tasks running in the background)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer.start_streaming(variables=[\"pot2\"])\n", + "streamer.plot_data(x_var=\"pot2\", y_vars=[\"pot2\"], y_range=[0, 1])\n", + "streamer.wait(10)\n", + "streamer.stop_streaming()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Scheduling streaming sessions\n", + "You can schedule a streaming session to start and stop at a specific time using the `schedule_streaming()` method. This method takes the same arguments as `start_streaming()`, but it also takes a `timestamps` and `durations` argument." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "latest_timestamp = streamer.get_latest_timestamp() # get the latest timestamp\n", + "sample_rate = streamer.sample_rate # get the sample rate\n", + "start_timestamp = latest_timestamp + sample_rate # start streaming 1 second after the latest timestamp\n", + "duration = sample_rate # stream for 2 seconds\n", + "\n", + "streamer.schedule_streaming(\n", + " variables=[\"pot1\", \"pot2\"],\n", + " timestamps=[start_timestamp, start_timestamp],\n", + " durations=[duration, duration],\n", + " saving_enabled=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Note on streaming variables assigned at low frequency rates\n", + "The data buffers sent from Bela have fixed sizes. The buffers will only be sent when they are full, unless you use the streaming with scheduling feature (explained below). If the variables you are streaming are assigned at too low rates, these buffers will take too long to fill up and the data will be either sent to python with a delay or not sent at all (if the buffer is never filled). For example, floats using dense timestamping are sent in buffers of 1024 values. If the float variable is assigned once every 12 milliseconds, filling a buffer will take 1024/(1/0.012) = 12.3 seconds. \n", + "Hence, the streaming mode is not ideal for variables assigned at low rates, but rather for variables that are assigned quite frequently (e.g. at audio rate). If you want to stream variables that are assigned at lower rates, you can use the streaming with scheduling feature, or monitor or log the variable instead." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Retrieving the data\n", + "You can access the data streamed in `streamer.streaming_buffers_data`. We can use the pandas data manipulation library for printing the data onto a table:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(streamer.streaming_buffers_data[\"pot2\"])\n", + "df.head() # head shows only the first 5 rows" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see, `streaming_buffers_data` only retrieves the variable values but not its timestamps. If you want to retrieve the timestamps, you can access `streaming_buffers_queue[\"pot2\"]`. This will return a list in which every item is a timestamped buffer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer.streaming_buffers_queue[\"pot2\"][0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the buffer `ref_timestamp` corresponds to the timestamp of the first value of the buffer (`streaming_buffers_queue[\"pot2\"][0][\"data\"][0]`). If the Bela Watcher is ticked once per analog frame (as it is the case in the `potentiometer` code) and the variable `pot2` is assigned also once per analog frame, the timestamps of the rest of the values in the data buffer correspond to the increasing timestamps:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data_timestamps = []\n", + "data_values = []\n", + "\n", + "def flatten_buffers_queue(_buffer_queue):\n", + " for _buffer in _buffer_queue:\n", + " ref_timestamp = _buffer[\"ref_timestamp\"]\n", + " data_timestamps.extend([ref_timestamp + i for i in range(len(_buffer[\"data\"]))])\n", + " data_values.extend(_buffer[\"data\"])\n", + " \n", + " return data_timestamps, data_values\n", + "\n", + "data_timestamps, data_values = flatten_buffers_queue(streamer.streaming_buffers_queue[\"pot2\"])\n", + " \n", + "df = pd.DataFrame({\"timestamp\": data_timestamps, \"value\": data_values})\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "More advanced timestamping methods will be shown in the tutorial notebook `7_Sparse_timestamping.ipynb`\n", + "\n", + "There is a limited amount of data that is stored in the streamer. This quantity can be modified by changing the buffer queue length. The streamer receives the data in buffers of fixed length that get stored in a queue that also has a fixed length. You can calculate the maximum amount of data the streamer can store for each variable:\n", + "\n", + "note: `streamer.watcher_vars` returns information of the variables available in the watcher, that is, variables that have been defined within the Watcher class in the Bela code and that are available for streaming, monitoring or logging." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Buffer queue length: {streamer.streaming_buffers_queue_length}\")\n", + "\n", + "for var in streamer.watcher_vars: \n", + " print(f'Variable: {var[\"name\"]}, buffer length: {var[\"data_length\"]}, max data stored in streamer: {var[\"data_length\"]*streamer.streaming_buffers_queue_length}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also modify the queue length:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer.streaming_buffers_queue_length = 10" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Saving the streamed data\n", + "Every time you start a new streaming session (e.g. you call `start_streaming()` or `stream_n_values()`), the data stored in the streamer from the previous streaming session will be deleted. If you want to store the streamed data, you can do so by setting `saving_enabled=True` when calling `start_streaming()` or `stream_n_values()`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer.start_streaming(variables=[var[\"name\"] for var in streamer.watcher_vars], saving_enabled=True, saving_filename=\"test.txt\")\n", + "streamer.wait(3)\n", + "streamer.stop_streaming()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can load the data stored using the `load_data_from_file` method. This will return the buffers queue. Again, we can flatten it using the `flatten_buffers_queue()` function we defined above:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data_timestamps, data_values = flatten_buffers_queue(streamer.load_data_from_file(\"pot1_test.txt\"))\n", + "\n", + "df=pd.DataFrame({\"timestamp\": data_timestamps, \"value\": data_values})\n", + "df.head()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pybela-2uXYSGIe", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.19" + } + }, + "nbformat": 4, + "nbformat_minor": 4 } diff --git a/tutorials/notebooks/2_Streamer-Bela-to-python-advanced.ipynb b/tutorials/notebooks/2_Streamer-Bela-to-python-advanced.ipynb index 342c708..73f54cb 100644 --- a/tutorials/notebooks/2_Streamer-Bela-to-python-advanced.ipynb +++ b/tutorials/notebooks/2_Streamer-Bela-to-python-advanced.ipynb @@ -1,208 +1,211 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# pybela Tutorial 2: Streamer – Bela to python advanced\n", - "This notebook is a tutorial for the Streamer class in the pybela python library. You can use the Streamer to stream data from Bela to python or vice versa. \n", - "\n", - "In this tutorial we will be looking at more advanced features to send data from Bela to python. \n", - "\n", - "The complete documentation for the pybela library can be found in [https://belaplatform.github.io/pybela/](https://belaplatform.github.io/pybela/).\n", - "\n", - "If you didn't do it in the previous tutorial, copy the `bela-code/python-to-bela` project onto Bela. If your Bela is connected to your laptop, you can run the cell below:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!rsync -rvL ../bela-code/python-to-bela root@bela.local:Bela/projects" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Then you can compile and run the project using either the IDE or by running the following command in the Terminal:\n", - "```bash\n", - "ssh root@bela.local \"make -C Bela stop Bela PROJECT=python-to-bela run\" \n", - "```\n", - "(Running this on a jupyter notebook will block the cell until the program is stopped on Bela.) You will also need to connect two potentiometers to Bela analog inputs 0 and 1. Instructions on how to do so and some details on the Bela code are given in the notebook `1_Streamer-Bela-to-python-basics.ipynb`.\n", - "\n", - "First, we need to import the pybela library, create a Streamer object and connect to Bela." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import asyncio\n", - "from pybela import Streamer\n", - "\n", - "streamer = Streamer()\n", - "streamer.connect()\n", - "\n", - "variables = [\"pot1\", \"pot2\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Streaming a fixed number of values\n", - "You can can use the method `stream_n_values` to stream a fixed number of values of a variable. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "n_values = 1000\n", - "streaming_buffer = streamer.stream_n_values(\n", - " variables=[var[\"name\"] for var in variables], n_values=n_values)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Since the data buffers received from Bela have a fixed size, unless the number of values `n_values` is a multiple of the data buffers size, the streamer will always return a few more values than asked for." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for var in variables:\n", - " print(f'Variable: {var[\"name\"]}, buffer length: {var[\"data_length\"]}, number of streamed values: {len(streamer.streaming_buffers_data[var[\"name\"]])}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Scheduling streaming sessions\n", - "You can schedule a streaming session to start and stop at a specific time using the `schedule_streaming()` method. This method takes the same arguments as `start_streaming()`, but it also takes a `timestamps` and `durations` argument." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "latest_timestamp = streamer.get_latest_timestamp() # get the latest timestamp\n", - "sample_rate = streamer.sample_rate # get the sample rate\n", - "start_timestamp = latest_timestamp + sample_rate # start streaming 1 second after the latest timestamp\n", - "duration = sample_rate # stream for 2 seconds\n", - "\n", - "streamer.schedule_streaming(\n", - " variables=variables,\n", - " timestamps=[start_timestamp, start_timestamp],\n", - " durations=[duration, duration],\n", - " saving_enabled=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### On-buffer and on-block callbacks\n", - "Up until now, we have been streaming data for a period of time and processed the data once the streaming has finished. However, you can also process the data as it is being received. You can do this by passing a callback function to the `on_buffer` or `on_block` arguments of the `start_streaming()` method. \n", - "\n", - "The `on_buffer` callback will be called every time a buffer is received from Bela. We will need to define a callback function that takes one argument, the buffer. The Streamer will call that function every time it receives a buffer. You can also pass variables to the callback function by using the `callback_args` argument of the `start_streaming()` method. Let's see an example:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "timestamps = {var: [] for var in variables}\n", - "buffers = {var: [] for var in variables}\n", - "\n", - "def callback(buffer, timestamps, buffers):\n", - " print(\"Buffer received\")\n", - " \n", - " _var = buffer[\"name\"]\n", - " timestamps[_var].append(\n", - " buffer[\"buffer\"][\"ref_timestamp\"])\n", - " buffers[_var].append(buffer[\"buffer\"][\"data\"])\n", - " \n", - " print(_var, timestamps[_var][-1])\n", - "\n", - "streamer.start_streaming(\n", - " variables, saving_enabled=False, on_buffer_callback=callback, callback_args=(timestamps, buffers))\n", - "\n", - "await asyncio.sleep(2)\n", - "streamer.stop_streaming()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's now look at the `on_block`callback. We call block to a group of buffers. If you are streaming two variables, `pot1` and `pot2`, a block of buffers will contain a buffer for `pot1` and a buffer for `pot2`. If `pot1` and `pot2` have the same buffer size and they are being streamed at the same rate, `pot1` and `pot2` will be aligned in time. This is useful if you are streaming multiple variables and you want to process them together. \n", - "\n", - "The `on_block` callback will be called every time a block of buffers is received from Bela. We will need to define a callback function that takes one argument, the block. The Streamer will call that function every time it receives a block of buffers. Let's see an example:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "timestamps = {var: [] for var in variables}\n", - "buffers = {var: [] for var in variables}\n", - "\n", - "def callback(block, timestamps, buffers):\n", - " print(\"Block received\")\n", - " \n", - " for buffer in block:\n", - " var = buffer[\"name\"]\n", - " timestamps[var].append(buffer[\"buffer\"][\"ref_timestamp\"])\n", - " buffers[var].append(buffer[\"buffer\"][\"data\"])\n", - "\n", - " print(var, timestamps[var][-1])\n", - " \n", - "streamer.start_streaming(\n", - " variables, saving_enabled=False, on_block_callback=callback, callback_args=(timestamps, buffers))\n", - "await asyncio.sleep(2)\n", - "streamer.stop_streaming()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "pybela-2uXYSGIe", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# pybela Tutorial 2: Streamer – Bela to python advanced\n", + "This notebook is a tutorial for the Streamer class in the pybela python library. You can use the Streamer to stream data from Bela to python or vice versa. \n", + "\n", + "In this tutorial we will be looking at more advanced features to send data from Bela to python. \n", + "\n", + "The complete documentation for the pybela library can be found in [https://belaplatform.github.io/pybela/](https://belaplatform.github.io/pybela/).\n", + "\n", + "If you didn't do it in the previous tutorial, copy the `bela-code/potentiometers` project onto Bela. If your Bela is connected to your laptop, you can run the cell below:" + ] }, - "nbformat": 4, - "nbformat_minor": 2 + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!rsync -rvL ../bela-code/potentiometers root@bela.local:Bela/projects" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then you can compile and run the project using either the IDE or by running the following command in the Terminal:\n", + "```bash\n", + "ssh root@bela.local \"make -C Bela stop Bela PROJECT=potentiometers run\" \n", + "```\n", + "(Running this on a jupyter notebook will block the cell until the program is stopped on Bela.) You will also need to connect two potentiometers to Bela analog inputs 0 and 1. Instructions on how to do so and some details on the Bela code are given in the notebook `1_Streamer-Bela-to-python-basics.ipynb`.\n", + "\n", + "First, we need to import the pybela library, create a Streamer object and connect to Bela." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pybela import Streamer\n", + "\n", + "streamer = Streamer()\n", + "streamer.connect()\n", + "\n", + "variables = [\"pot1\", \"pot2\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming a fixed number of values\n", + "You can can use the method `stream_n_values` to stream a fixed number of values of a variable. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "n_values = 1000\n", + "streaming_buffer = streamer.stream_n_values(\n", + " variables= variables, n_values=n_values)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Since the data buffers received from Bela have a fixed size, unless the number of values `n_values` is a multiple of the data buffers size, the streamer will always return a few more values than asked for." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "_vars = streamer.watcher_vars\n", + "for var in _vars:\n", + " print(f'Variable: {var[\"name\"]}, buffer length: {var[\"data_length\"]}, number of streamed values: {len(streamer.streaming_buffers_data[var[\"name\"]])}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Scheduling streaming sessions\n", + "You can schedule a streaming session to start and stop at a specific time using the `schedule_streaming()` method. This method takes the same arguments as `start_streaming()`, but it also takes a `timestamps` and `durations` argument." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "latest_timestamp = streamer.get_latest_timestamp() # get the latest timestamp\n", + "sample_rate = streamer.sample_rate # get the sample rate\n", + "start_timestamp = latest_timestamp + sample_rate # start streaming 1 second after the latest timestamp\n", + "duration = sample_rate # stream for 2 seconds\n", + "\n", + "streamer.schedule_streaming(\n", + " variables=variables,\n", + " timestamps=[start_timestamp, start_timestamp],\n", + " durations=[duration, duration],\n", + " saving_enabled=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### On-buffer and on-block callbacks\n", + "Up until now, we have been streaming data for a period of time and processed the data once the streaming has finished. However, you can also process the data as it is being received. You can do this by passing a callback function to the `on_buffer` or `on_block` arguments of the `start_streaming()` method. \n", + "\n", + "The `on_buffer` callback will be called every time a buffer is received from Bela. We will need to define a callback function that takes one argument, the buffer. The Streamer will call that function every time it receives a buffer. You can also pass variables to the callback function by using the `callback_args` argument of the `start_streaming()` method. Let's see an example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "timestamps = {var: [] for var in variables}\n", + "buffers = {var: [] for var in variables}\n", + "\n", + "def callback(buffer, timestamps, buffers):\n", + " print(\"Buffer received\")\n", + " \n", + " _var = buffer[\"name\"]\n", + " timestamps[_var].append(\n", + " buffer[\"buffer\"][\"ref_timestamp\"])\n", + " buffers[_var].append(buffer[\"buffer\"][\"data\"])\n", + " \n", + " print(_var, timestamps[_var][-1])\n", + "\n", + "streamer.start_streaming(\n", + " variables, saving_enabled=False, on_buffer_callback=callback, callback_args=(timestamps, buffers))\n", + "\n", + "streamer.wait(2)\n", + "\n", + "streamer.stop_streaming()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's now look at the `on_block`callback. We call block to a group of buffers. If you are streaming two variables, `pot1` and `pot2`, a block of buffers will contain a buffer for `pot1` and a buffer for `pot2`. If `pot1` and `pot2` have the same buffer size and they are being streamed at the same rate, `pot1` and `pot2` will be aligned in time. This is useful if you are streaming multiple variables and you want to process them together. \n", + "\n", + "The `on_block` callback will be called every time a block of buffers is received from Bela. We will need to define a callback function that takes one argument, the block. The Streamer will call that function every time it receives a block of buffers. Let's see an example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "timestamps = {var: [] for var in variables}\n", + "buffers = {var: [] for var in variables}\n", + "\n", + "def callback(block, timestamps, buffers):\n", + " print(\"Block received\")\n", + " \n", + " for buffer in block:\n", + " var = buffer[\"name\"]\n", + " timestamps[var].append(buffer[\"buffer\"][\"ref_timestamp\"])\n", + " buffers[var].append(buffer[\"buffer\"][\"data\"])\n", + "\n", + " print(var, timestamps[var][-1])\n", + " \n", + "streamer.start_streaming(\n", + " variables, saving_enabled=False, on_block_callback=callback, callback_args=(timestamps, buffers))\n", + "\n", + "streamer.wait(2)\n", + "\n", + "streamer.stop_streaming()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pybela-2uXYSGIe", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.19" + } + }, + "nbformat": 4, + "nbformat_minor": 2 } diff --git a/tutorials/notebooks/3_Streamer-python-to-Bela.ipynb b/tutorials/notebooks/3_Streamer-python-to-Bela.ipynb index 387622e..fa33729 100644 --- a/tutorials/notebooks/3_Streamer-python-to-Bela.ipynb +++ b/tutorials/notebooks/3_Streamer-python-to-Bela.ipynb @@ -1,245 +1,240 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# pybela Tutorial 3: Streamer – python to Bela\n", - "This notebook is a tutorial for the Streamer class in the pybela python library. You can use the Streamer to stream data from Bela to python or viceversa. The complete documentation for the pybela library can be found in [https://belaplatform.github.io/pybela/](https://belaplatform.github.io/pybela/).\n", - "\n", - "In this tutorial we will be looking at sending data from python to Bela. There is only one method available in the Streamer class for this purpose: `send_buffer()`. This method sends a buffer of a certain type and size to Bela. \n", - "\n", - "To run this tutorial, first copy the `bela-code/bela2python2bela` project onto Bela. If your Bela is connected to your laptop, you can run the cell below:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!rsync -rvL ../bela-code/bela2python2bela root@bela.local:Bela/projects" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Then you can compile and run the project using either the IDE or by running the following command in the Terminal:\n", - "```bash\n", - "ssh root@bela.local \"make -C Bela stop Bela PROJECT=bela2python2bela run\" \n", - "```\n", - "(Running this on a jupyter notebook will block the cell until the program is stopped on Bela.) \n", - "\n", - "This program expects two analog signals in channels 0 and 1, you can keep using the potentiometer setup from the previous tutorials (check the schematic in `1_Streamer-Bela-to-python.ipynb`)\n", - "\n", - "In this example we will be sending the values of the two potentiometers from Bela to python. Once received in python, we will send them immediately back to Bela. The values received in Bela will be used to modulate the amplitude of two sine waves. It is admittedly an overly complicated way to modulate two sine waves in Bela, as you could of course use the potentiometer values directly, without having to send them to python and back. However, this example can serve as a template for more complex applications where you can process the data in python before sending it back to Bela. \n", - "\n", - "## Understanding the Bela code\n", - "If you are not familiar with auxiliary tasks and circular buffers, we recommend you follow first [Lesson 11](https://youtu.be/xQBftd7WNY8?si=ns6ojYnfQ_GVtCQI) and [Lesson 17](https://youtu.be/2uyWn8P0CVg?si=Ymy-NN_HKS-Q3xL0) of the C++ Real-Time Audio Programming with Bela course. \n", - "\n", - "Let's first take a look at the Bela code. The `setup()` function initializes the Bela program and some necessary variables. First, we set up the Watcher with the `Bela_getDefaultWatcherManager()` function. We then calculate the inverse of some useful variables (multiplying by the inverse is faster than dividing, so we precompute the inverse in `setup` and use it later in `render`). We then initialize the GUI buffers (these are the internal buffers Bela uses to receive the data) and the `circularBuffers`. The `circularBuffers` are used to store the parsed data from the GUI buffers, and are the variables we will use in `render` to access the data we have sent from python. We also set up the `binaryDataCallback` function, which will be called when Bela receives a buffer from python. \n", - "\n", - "\n", - "```cpp\n", - "bool setup(BelaContext* context, void* userData) {\n", - "\n", - " Bela_getDefaultWatcherManager()->getGui().setup(context->projectName);\n", - " Bela_getDefaultWatcherManager()->setup(context->audioSampleRate); // set sample rate in watcher\n", - "\n", - " gAudioFramesPerAnalogFrame = context->audioFrames / context->analogFrames;\n", - " gInvAudioFramesPerAnalogFrame = 1.0 / gAudioFramesPerAnalogFrame;\n", - " gInverseSampleRate = 1.0 / context->audioSampleRate;\n", - "\n", - " // initialize the Gui buffers and circular buffers\n", - " for (int i = 0; i < NUM_OUTPUTS; ++i) {\n", - " Bela_getDefaultWatcherManager()->getGui().setBuffer('f', MAX_EXPECTED_BUFFER_SIZE);\n", - " circularBuffers[i].resize(circularBufferSize, 0.0f);\n", - " // the write index is given some \"advantage\" (prefillSize) so that the read pointer does not catch up the write pointer\n", - " circularBufferWriteIndex[i] = prefillSize % circularBufferSize;\n", - " }\n", - "\n", - " Bela_getDefaultWatcherManager()->getGui().setBinaryDataCallback(binaryDataCallback);\n", - "\n", - " // vars and preparation for parsing the received buffer\n", - " receivedBufferHeaderSize = sizeof(receivedBuffer.bufferId) + sizeof(receivedBuffer.bufferType) + sizeof(receivedBuffer.bufferLen) + sizeof(receivedBuffer.empty);\n", - " totalReceivedCount = 0;\n", - " receivedBuffer.bufferData.reserve(MAX_EXPECTED_BUFFER_SIZE);\n", - "\n", - " return true;\n", - "}\n", - "```\n", - "\n", - "Let's now take a look at the `render()` function. The render function is called once per audio block, so inside of it we iterate over the audio blocks. Since the potentiometers are analog signals, and in Bela the analog inputs are typically sampled at a lower rate than the audio, we read the potentiometers once every 2 audio frames (in the code, `gAudioFramesPerAnalogFrame` is equal to 2 if you are using the default 8 audio channels). Since the variables `pot1` and `pot2` are in the Watcher, these will be streamed to python if we run `start_streaming()` in python.\n", - "\n", - "Next, we check if the variable `totalReceivedCount` is greater than 0, which means that we have received at least a buffer from python. If we have received buffers and the read pointer has not caught up with the write pointer, we advance the read pointer in the circular buffer. The reason why we check if we have received a buffer first, is because we don't want to advance the read pointer if we haven't received any data yet, as then the read pointer would catch up with the write pointer. \n", - "\n", - "Finally, we read the values from the circular buffer and use them to modulate the amplitude of two sine waves. We then write the output to the audio channels.\n", - "\n", - "\n", - "\n", - "```cpp\n", - "\n", - "void render(BelaContext* context, void* userData) {\n", - " for (unsigned int n = 0; n < context->audioFrames; n++) {\n", - " uint64_t frames = context->audioFramesElapsed + n;\n", - "\n", - " if (gAudioFramesPerAnalogFrame && !(n % gAudioFramesPerAnalogFrame)) {\n", - " Bela_getDefaultWatcherManager()->tick(frames * gInvAudioFramesPerAnalogFrame); // watcher timestamps\n", - "\n", - " // read sensor values and put them in the watcher\n", - " pot1 = analogRead(context, n / gAudioFramesPerAnalogFrame, gPot1Ch);\n", - " pot2 = analogRead(context, n / gAudioFramesPerAnalogFrame, gPot2Ch);\n", - "\n", - " // read the values sent from python (they're in the circular buffer)\n", - " for (unsigned int i = 0; i < NUM_OUTPUTS; i++) {\n", - "\n", - " if (totalReceivedCount > 0 && (circularBufferReadIndex[i] + 1) % circularBufferSize != circularBufferWriteIndex[i]) {\n", - " circularBufferReadIndex[i] = (circularBufferReadIndex[i] + 1) % circularBufferSize;\n", - " } else if (totalReceivedCount > 0) {\n", - " rt_printf(\"The read pointer has caught the write pointer up in buffer %d – try increasing prefillSize\\n\", i);\n", - " }\n", - " }\n", - " }\n", - "\n", - " float amp1 = circularBuffers[0][circularBufferReadIndex[0]];\n", - " float amp2 = circularBuffers[1][circularBufferReadIndex[1]];\n", - "\n", - " float out = amp1 * sinf(gPhase1) + amp2 * sinf(gPhase2);\n", - "\n", - " for (unsigned int channel = 0; channel < context->audioOutChannels; channel++) {\n", - " audioWrite(context, n, channel, out);\n", - " }\n", - "\n", - " gPhase1 += 2.0f * (float)M_PI * gFrequency1 * gInverseSampleRate;\n", - " if (gPhase1 > M_PI)\n", - " gPhase1 -= 2.0f * (float)M_PI;\n", - " gPhase2 += 2.0f * (float)M_PI * gFrequency2 * gInverseSampleRate;\n", - " if (gPhase2 > M_PI)\n", - " gPhase2 -= 2.0f * (float)M_PI;\n", - "\n", - " }\n", - "}\n", - "```\n", - "\n", - "Let's now run the python code:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from pybela import Streamer\n", - "streamer = Streamer()\n", - "streamer.connect()\n", - "\n", - "variables = [\"pot1\", \"pot2\"]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `send_buffer` function takes 4 arguments: the buffer id, the type of the data that goes in the buffer, the buffer length and the buffer data. Since we will be sending back the buffers we receive from Bela, we can get the type and length of the buffer through the streamer:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "buffer_type = streamer.get_prop_of_var(\"pot1\", \"type\")\n", - "buffer_length = streamer.get_prop_of_var(\"pot1\", \"data_length\")\n", - "\n", - "buffer_type, buffer_length\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here we will be using the `block_callback` instead of the `buffer_callback`, as the `block` callback is more efficient. It should be noted that we are receiving and sending blocks of data every 1024/22050 = 0.05 seconds, and the maximum latency is given by the `prefillSize` variable in the Bela code (which is set to 2.5*1024/22050 = 0.12 seconds), so using functions is crucial to meet the real-time deadlines." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def callback(block):\n", - " \n", - " for buffer in block:\n", - " \n", - " _var = buffer[\"name\"]\n", - " timestamp = buffer[\"buffer\"][\"ref_timestamp\"]\n", - " data = buffer[\"buffer\"][\"data\"]\n", - " \n", - " buffer_id = 0 if _var == \"pot1\" else 1\n", - "\n", - " print(buffer_id, timestamp)\n", - " # do some data processing here...\n", - " processed_data = data\n", - " \n", - " # send processed_data back\n", - " streamer.send_buffer(buffer_id, buffer_type,\n", - " buffer_length, processed_data)\n", - "\n", - "streamer.start_streaming(\n", - " variables, saving_enabled=False, on_block_callback=callback)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If you plug in your headphones to the audio output of Bela, you should hear two sine waves modulated by the potentiometers. The modulation (the amplitude change) is given by the value sent by python, not the analog input directly on Bela. As mentioned before, this is an overly complicated way to modulate two sine waves, but it can serve as a template for more complex applications where you can process the data in python before sending it back to Bela." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "..." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "streamer.stop_streaming()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.16" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# pybela Tutorial 3: Streamer – python to Bela\n", + "This notebook is a tutorial for the Streamer class in the pybela python library. You can use the Streamer to stream data from Bela to python or viceversa. The complete documentation for the pybela library can be found in [https://belaplatform.github.io/pybela/](https://belaplatform.github.io/pybela/).\n", + "\n", + "In this tutorial we will be looking at sending data from python to Bela. There is only one method available in the Streamer class for this purpose: `send_buffer()`. This method sends a buffer of a certain type and size to Bela. \n", + "\n", + "To run this tutorial, first copy the `bela-code/bela2python2bela` project onto Bela. If your Bela is connected to your laptop, you can run the cell below:" + ] }, - "nbformat": 4, - "nbformat_minor": 4 + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!rsync -rvL ../bela-code/bela2python2bela root@bela.local:Bela/projects" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then you can compile and run the project using either the IDE or by running the following command in the Terminal:\n", + "```bash\n", + "ssh root@bela.local \"make -C Bela stop Bela PROJECT=bela2python2bela run\" \n", + "```\n", + "(Running this on a jupyter notebook will block the cell until the program is stopped on Bela.) \n", + "\n", + "This program expects two analog signals in channels 0 and 1, you can keep using the potentiometer setup from the previous tutorials (check the schematic in `1_Streamer-Bela-to-python.ipynb`)\n", + "\n", + "In this example we will be sending the values of the two potentiometers from Bela to python. Once received in python, we will send them immediately back to Bela. The values received in Bela will be used to modulate the amplitude of two sine waves. It is admittedly an overly complicated way to modulate two sine waves in Bela, as you could of course use the potentiometer values directly, without having to send them to python and back. However, this example can serve as a template for more complex applications where you can process the data in python before sending it back to Bela. \n", + "\n", + "## Understanding the Bela code\n", + "If you are not familiar with auxiliary tasks and circular buffers, we recommend you follow first [Lesson 11](https://youtu.be/xQBftd7WNY8?si=ns6ojYnfQ_GVtCQI) and [Lesson 17](https://youtu.be/2uyWn8P0CVg?si=Ymy-NN_HKS-Q3xL0) of the C++ Real-Time Audio Programming with Bela course. \n", + "\n", + "Let's first take a look at the Bela code. The `setup()` function initializes the Bela program and some necessary variables. First, we set up the Watcher with the `Bela_getDefaultWatcherManager()` function. We then calculate the inverse of some useful variables (multiplying by the inverse is faster than dividing, so we precompute the inverse in `setup` and use it later in `render`). We then initialize the GUI buffers (these are the internal buffers Bela uses to receive the data) and the `circularBuffers`. The `circularBuffers` are used to store the parsed data from the GUI buffers, and are the variables we will use in `render` to access the data we have sent from python. We also set up the `binaryDataCallback` function, which will be called when Bela receives a buffer from python. \n", + "\n", + "\n", + "```cpp\n", + "bool setup(BelaContext* context, void* userData) {\n", + "\n", + " Bela_getDefaultWatcherManager()->getGui().setup(context->projectName);\n", + " Bela_getDefaultWatcherManager()->setup(context->audioSampleRate); // set sample rate in watcher\n", + "\n", + " gAudioFramesPerAnalogFrame = context->audioFrames / context->analogFrames;\n", + " gInvAudioFramesPerAnalogFrame = 1.0 / gAudioFramesPerAnalogFrame;\n", + " gInverseSampleRate = 1.0 / context->audioSampleRate;\n", + "\n", + " // initialize the Gui buffers and circular buffers\n", + " for (int i = 0; i < NUM_OUTPUTS; ++i) {\n", + " Bela_getDefaultWatcherManager()->getGui().setBuffer('f', MAX_EXPECTED_BUFFER_SIZE);\n", + " circularBuffers[i].resize(circularBufferSize, 0.0f);\n", + " // the write index is given some \"advantage\" (prefillSize) so that the read pointer does not catch up the write pointer\n", + " circularBufferWriteIndex[i] = prefillSize % circularBufferSize;\n", + " }\n", + "\n", + " Bela_getDefaultWatcherManager()->getGui().setBinaryDataCallback(binaryDataCallback);\n", + "\n", + " // vars and preparation for parsing the received buffer\n", + " receivedBufferHeaderSize = sizeof(receivedBuffer.bufferId) + sizeof(receivedBuffer.bufferType) + sizeof(receivedBuffer.bufferLen) + sizeof(receivedBuffer.empty);\n", + " totalReceivedCount = 0;\n", + " receivedBuffer.bufferData.reserve(MAX_EXPECTED_BUFFER_SIZE);\n", + "\n", + " return true;\n", + "}\n", + "```\n", + "\n", + "Let's now take a look at the `render()` function. The render function is called once per audio block, so inside of it we iterate over the audio blocks. Since the potentiometers are analog signals, and in Bela the analog inputs are typically sampled at a lower rate than the audio, we read the potentiometers once every 2 audio frames (in the code, `gAudioFramesPerAnalogFrame` is equal to 2 if you are using the default 8 audio channels). Since the variables `pot1` and `pot2` are in the Watcher, these will be streamed to python if we run `start_streaming()` in python.\n", + "\n", + "Next, we check if the variable `totalReceivedCount` is greater than 0, which means that we have received at least a buffer from python. If we have received buffers and the read pointer has not caught up with the write pointer, we advance the read pointer in the circular buffer. The reason why we check if we have received a buffer first, is because we don't want to advance the read pointer if we haven't received any data yet, as then the read pointer would catch up with the write pointer. \n", + "\n", + "Finally, we read the values from the circular buffer and use them to modulate the amplitude of two sine waves. We then write the output to the audio channels.\n", + "\n", + "\n", + "\n", + "```cpp\n", + "\n", + "void render(BelaContext* context, void* userData) {\n", + " for (unsigned int n = 0; n < context->audioFrames; n++) {\n", + " uint64_t frames = context->audioFramesElapsed + n;\n", + "\n", + " if (gAudioFramesPerAnalogFrame && !(n % gAudioFramesPerAnalogFrame)) {\n", + " Bela_getDefaultWatcherManager()->tick(frames * gInvAudioFramesPerAnalogFrame); // watcher timestamps\n", + "\n", + " // read sensor values and put them in the watcher\n", + " pot1 = analogRead(context, n / gAudioFramesPerAnalogFrame, gPot1Ch);\n", + " pot2 = analogRead(context, n / gAudioFramesPerAnalogFrame, gPot2Ch);\n", + "\n", + " // read the values sent from python (they're in the circular buffer)\n", + " for (unsigned int i = 0; i < NUM_OUTPUTS; i++) {\n", + "\n", + " if (totalReceivedCount > 0 && (circularBufferReadIndex[i] + 1) % circularBufferSize != circularBufferWriteIndex[i]) {\n", + " circularBufferReadIndex[i] = (circularBufferReadIndex[i] + 1) % circularBufferSize;\n", + " } else if (totalReceivedCount > 0) {\n", + " rt_printf(\"The read pointer has caught the write pointer up in buffer %d – try increasing prefillSize\\n\", i);\n", + " }\n", + " }\n", + " }\n", + "\n", + " float amp1 = circularBuffers[0][circularBufferReadIndex[0]];\n", + " float amp2 = circularBuffers[1][circularBufferReadIndex[1]];\n", + "\n", + " float out = amp1 * sinf(gPhase1) + amp2 * sinf(gPhase2);\n", + "\n", + " for (unsigned int channel = 0; channel < context->audioOutChannels; channel++) {\n", + " audioWrite(context, n, channel, out);\n", + " }\n", + "\n", + " gPhase1 += 2.0f * (float)M_PI * gFrequency1 * gInverseSampleRate;\n", + " if (gPhase1 > M_PI)\n", + " gPhase1 -= 2.0f * (float)M_PI;\n", + " gPhase2 += 2.0f * (float)M_PI * gFrequency2 * gInverseSampleRate;\n", + " if (gPhase2 > M_PI)\n", + " gPhase2 -= 2.0f * (float)M_PI;\n", + "\n", + " }\n", + "}\n", + "```\n", + "\n", + "Let's now run the python code:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pybela import Streamer\n", + "streamer = Streamer()\n", + "streamer.connect()\n", + "\n", + "variables = [\"pot1\", \"pot2\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `send_buffer` function takes 4 arguments: the buffer id, the type of the data that goes in the buffer, the buffer length and the buffer data. Since we will be sending back the buffers we receive from Bela, we can get the type and length of the buffer through the streamer:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "buffer_type = streamer.get_prop_of_var(\"pot1\", \"type\")\n", + "buffer_length = streamer.get_prop_of_var(\"pot1\", \"data_length\")\n", + "\n", + "buffer_type, buffer_length\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we will be using the `block_callback` instead of the `buffer_callback`, as the `block` callback is more efficient. It should be noted that we are receiving and sending blocks of data every 1024/22050 = 0.05 seconds, and the maximum latency is given by the `prefillSize` variable in the Bela code (which is set to 2.5*1024/22050 = 0.12 seconds), so using functions is crucial to meet the real-time deadlines." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def callback(block):\n", + " \n", + " for buffer in block:\n", + " \n", + " _var = buffer[\"name\"]\n", + " timestamp = buffer[\"buffer\"][\"ref_timestamp\"]\n", + " data = buffer[\"buffer\"][\"data\"]\n", + " \n", + " buffer_id = 0 if _var == \"pot1\" else 1\n", + "\n", + " print(buffer_id, timestamp)\n", + " # do some data processing here...\n", + " processed_data = data\n", + " \n", + " # send processed_data back\n", + " streamer.send_buffer(buffer_id, buffer_type,\n", + " buffer_length, processed_data)\n", + "\n", + "streamer.start_streaming(\n", + " variables, saving_enabled=False, on_block_callback=callback)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you plug in your headphones to the audio output of Bela, you should hear two sine waves modulated by the potentiometers. The modulation (the amplitude change) is given by the value sent by python, not the analog input directly on Bela. As mentioned before, this is an overly complicated way to modulate two sine waves, but it can serve as a template for more complex applications where you can process the data in python before sending it back to Bela." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "..." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer.stop_streaming()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pybela-2uXYSGIe", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.19" + } + }, + "nbformat": 4, + "nbformat_minor": 4 } diff --git a/tutorials/notebooks/4_Monitor.ipynb b/tutorials/notebooks/4_Monitor.ipynb index 5602d99..e52c24b 100644 --- a/tutorials/notebooks/4_Monitor.ipynb +++ b/tutorials/notebooks/4_Monitor.ipynb @@ -40,15 +40,8 @@ "metadata": {}, "outputs": [], "source": [ - "import asyncio\n", - "import os\n", "import pandas as pd\n", - "from pybela import Monitor\n", - "\n", - "# this environment variable allows displaying the bokeh plots. If you are in vscode, you need to set this to the vscode port:\n", - "os.environ[\"BOKEH_ALLOW_WS_ORIGIN\"] =\"1t4j54lsdj67h02ol8hionopt4k7b7ngd9483l5q5pagr3j2droq\" \n", - "# alternatively, you can set it to the jupyter notebook port:\n", - "# os.environ[\"BOKEH_ALLOW_WS_ORIGIN\"]=\"127.0.0.1:8888\" # if in jupyter notebook -- replace with the port number of your notebook" + "from pybela import Monitor" ] }, { @@ -87,7 +80,9 @@ "source": [ "monitor.start_monitoring(variables= [\"pot1\", \"pot2\"], \n", " periods= [1000,2000])\n", - "await asyncio.sleep(2)\n", + "\n", + "monitor.wait(2)\n", + "\n", "monitored_values = monitor.stop_monitoring()" ] }, @@ -219,7 +214,7 @@ "outputs": [], "source": [ "monitor.start_monitoring(variables= [\"pot1\", \"pot2\"],periods= [600,2000], saving_enabled=True)\n", - "await asyncio.sleep(1)\n", + "monitor.wait(2)\n", "monitor.stop_monitoring()" ] }, @@ -263,7 +258,7 @@ ], "metadata": { "kernelspec": { - "display_name": "pybela-irbKdG5b", + "display_name": "pybela-2uXYSGIe", "language": "python", "name": "python3" }, @@ -277,7 +272,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.8" + "version": "3.9.19" }, "orig_nbformat": 4 }, diff --git a/tutorials/notebooks/5_Logger.ipynb b/tutorials/notebooks/5_Logger.ipynb index 853ad24..8cc275a 100644 --- a/tutorials/notebooks/5_Logger.ipynb +++ b/tutorials/notebooks/5_Logger.ipynb @@ -47,7 +47,6 @@ "metadata": {}, "outputs": [], "source": [ - "import asyncio\n", "import os\n", "import pandas as pd\n", "from pybela import Logger" @@ -80,7 +79,7 @@ "source": [ "file_paths = logger.start_logging(\n", " variables=[\"pot1\", \"pot2\"])\n", - "await asyncio.sleep(0.5)\n", + "logger.wait(0.5)\n", "logger.stop_logging()" ] }, @@ -207,7 +206,7 @@ "source": [ "file_paths = logger.start_logging(\n", " variables= [\"pot1\", \"pot2\"], transfer=False)\n", - "await asyncio.sleep(0.5)\n", + "logger.wait(0.5)\n", "logger.stop_logging()\n", "\n", "file_paths" @@ -249,7 +248,7 @@ ], "metadata": { "kernelspec": { - "display_name": "pybela-irbKdG5b", + "display_name": "pybela-2uXYSGIe", "language": "python", "name": "python3" }, @@ -263,7 +262,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.9.19" }, "orig_nbformat": 4 }, diff --git a/tutorials/notebooks/6_Controller.ipynb b/tutorials/notebooks/6_Controller.ipynb index 8ae8270..652556c 100644 --- a/tutorials/notebooks/6_Controller.ipynb +++ b/tutorials/notebooks/6_Controller.ipynb @@ -155,7 +155,7 @@ ], "metadata": { "kernelspec": { - "display_name": "pybela-rZFcaNs6", + "display_name": "pybela-2uXYSGIe", "language": "python", "name": "python3" }, @@ -169,7 +169,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.9.19" } }, "nbformat": 4, diff --git a/tutorials/notebooks/7_Sparse-timestamping.ipynb b/tutorials/notebooks/7_Sparse-timestamping.ipynb index 41c0272..d580fa4 100644 --- a/tutorials/notebooks/7_Sparse-timestamping.ipynb +++ b/tutorials/notebooks/7_Sparse-timestamping.ipynb @@ -1 +1,240 @@ -{"cells":[{"cell_type":"markdown","metadata":{},"source":["# pybela Tutorial 7: Sparse timestamping\n","In the potentiometer example used in the previous tutorials, the values for `pot1` and `pot2` are assigned at every audio frame. Let's take a look again at the `render()` loop (the Bela code for this example can be found in (in `bela-code/potentiometers/render.cpp`).\n","\n","```cpp\n","void render(BelaContext *context, void *userData)\n","{\n","\tfor(unsigned int n = 0; n < context->audioFrames; n++) {\n","\t\tif(gAudioFramesPerAnalogFrame && !(n % gAudioFramesPerAnalogFrame)) {\n","\t\t\t\n","\t\t\tuint64_t frames = context->audioFramesElapsed/gAudioFramesPerAnalogFrame + n/gAudioFramesPerAnalogFrame;\n","\t\t\tBela_getDefaultWatcherManager()->tick(frames); // watcher timestamps\n","\t\t\t\n","\t\t\tpot1 = analogRead(context, n/gAudioFramesPerAnalogFrame, gPot1Ch);\n","\t\t\tpot2 = analogRead(context, n/gAudioFramesPerAnalogFrame, gPot2Ch);\n","\t\t\t\n","\t\t}\n","\t}\n","}\n","```\n","\n","\n","The Watched clock is also \"ticked\" at every analog frame, so that the timestamps in the data correspond to the audio frames in the Bela code. The data buffers we received from Bela in the Streamer and the Logger had this form: `{\"ref_timestamp\": 92381, \"data\":[0.34, 0.45, ...]}`. Each data point is registered in the buffer every time we assign a value to `pot1` and `pot2` in the Bela code. The `ref_timestamp` corresponds to the timestamp of the first sample in the `data` array, in this case `0.34`. Since in the Bela code, we assign `pot1` and `pot2` at every audio frame, we can infer the timestamps of each value in the data array by incrementing `ref_timestamp` by 1 for each sample. \n","\n","This is an efficient way of storing data since instead of storing the timestamp of every item in the data array, we only store the timestamp of the first item. We call this *dense* timestamping. However, for many applications, we might not assign a value to a variable every frame, we might do it more than once per frame, once every few frames, or we might want to do it at irregular intervals. In these cases, we need to store the timestamp of every item in the data array. We call this *sparse* timestamping.\n","\n","In this tutorial we take a look at *sparse* timestamping. The complete documentation for the pybela library can be found in [https://belaplatform.github.io/pybela/](https://belaplatform.github.io/pybela/).\n","\n","First, transfer the Bela code we will use in this tutorial to Bela:\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["!rsync -rvL ../bela-code/timestamping root@bela.local:Bela/projects"]},{"cell_type":"markdown","metadata":{},"source":["Then you can compile and run the project using either the IDE or by running the following command in the Terminal:\n","```bash\n","ssh root@bela.local \"make -C Bela stop Bela PROJECT=potentiometers run\" \n","```\n","(Running this on a jupyter notebook will block the cell until the program is stopped on Bela.)"]},{"cell_type":"markdown","metadata":{},"source":["As in the previous tutorials, we will use two potentiometers connected to Bela analog inputs 0 and 1. Check the `1_Streamer.ipnyb` tutorial notebook for instructions on how to set up the circuit. \n","\n","### Bela C++ code\n","\n","\n","First, let's take a look at the Bela code. First, we have added `WatcherManager::kTimestampSample` to the declaration of `pot2`. This informs the Bela Watcher that `pot2` will be watched sparsely, that is, that the watcher will store a timestamp for every value assigned to `pot2`:\n","\n","```cpp\n","Watcher pot1(\"pot1\");\n","Watcher pot2(\"pot2\", WatcherManager::kTimestampSample);\n","```\n","\n","Now let's take a look at `render()`:\n","\n","```cpp\n","void render(BelaContext *context, void *userData)\n","{\n","\tfor(unsigned int n = 0; n < context->audioFrames; n++) {\n","\t\tif(gAudioFramesPerAnalogFrame && !(n % gAudioFramesPerAnalogFrame)) {\n","\t\t\t\n","\t\t\tuint64_t frames = context->audioFramesElapsed/gAudioFramesPerAnalogFrame + n/gAudioFramesPerAnalogFrame;\n","\t\t\tBela_getDefaultWatcherManager()->tick(frames); // watcher timestamps\n","\t\t\t\n","\t\t\tpot1 = analogRead(context, n/gAudioFramesPerAnalogFrame, gPot1Ch);\n","\n","\t\t\tif (frames % 12==0){\n","\t\t\t\tpot2 = analogRead(context, n/gAudioFramesPerAnalogFrame, gPot2Ch);\n","\t\t\t}\n","\t\t}\n","\t}\n","}\n","```\n","\n","We are \"ticking\" the Bela Watcher once per analog frame, so that the timestamps in the data correspond to the analog frames in the Bela code. We are assigning a value to `pot1` at every analog frame, as in the previous examples, but we are now only assigning a value to `pot2` every 12 frames. \n","\n","### Dealing with sparse timestamps in Python\n","\n","Let's now take a look at the data we receive from Bela. We will use the Streamer. Run the cells below to declare and connect the Streamer to Bela:"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["import asyncio\n","import pandas as pd\n","from pybela import Streamer"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["streamer = Streamer()\n","streamer.connect()"]},{"cell_type":"markdown","metadata":{},"source":["We can call `.list()` to take a look at the variables available to be streamed, their types and timestamp mode:"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["streamer.list()"]},{"cell_type":"markdown","metadata":{},"source":["`timestampMode` indicates if the timestamping is *sparse* (1) or *dense* (0). Now let's stream the data from Bela. We will stream `pot1` and `pot2`:"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["streamer.start_streaming(variables=[\"pot1\", \"pot2\"], saving_enabled=False)\n","await asyncio.sleep(2)\n","streamer.stop_streaming()"]},{"cell_type":"markdown","metadata":{},"source":["Now let's take a look at the streamed buffers for \"pot2\". Each buffer has the form `{\"ref_timestamp\": 912831, \"data\":[0.23, 0.24, ...], \"rel_timestamps\":[ 0, 12, ...]}`. `ref_timestamp` corresponds, as in the dense case, to the timestamp of the first data point in the `data` array. `rel_timestamps` is an array of timestamps relative to `ref_timestamp`. In this case, since we are assigning a value to `pot2` every 12 frames, the timestamps in `rel_timestamps` are `[0, 12, 24, 36, etc.]`."]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["streamer.streaming_buffers_queue[\"pot2\"]"]},{"cell_type":"markdown","metadata":{},"source":["You can now calculate the absolute timestamps of each data point by adding the values in `rel_timestamps` to `ref_timestamp`:"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["[streamer.streaming_buffers_queue[\"pot2\"][0][\"ref_timestamp\"]]*len(streamer.streaming_buffers_queue[\"pot2\"][0][\"rel_timestamps\"]) + streamer.streaming_buffers_queue[\"pot2\"][0][\"rel_timestamps\"]"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["pot2_data = {\"timestamps\":[], \"data\":[]}\n","\n","for _buffer in streamer.streaming_buffers_queue[\"pot2\"]:\n"," pot2_data[\"timestamps\"].extend([_buffer[\"ref_timestamp\"] + i for i in _buffer[\"rel_timestamps\"]])\n"," pot2_data[\"data\"].extend(_buffer[\"data\"])"]},{"cell_type":"markdown","metadata":{},"source":["Note that the timestamps are spaced by 12, as expected:"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["df = pd.DataFrame(pot2_data)\n","df.head()"]},{"cell_type":"markdown","metadata":{},"source":[]}],"metadata":{"kernelspec":{"display_name":"pybela-irbKdG5b","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.16"},"orig_nbformat":4},"nbformat":4,"nbformat_minor":2} +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# pybela Tutorial 7: Sparse timestamping\n", + "In the potentiometer example used in the previous tutorials, the values for `pot1` and `pot2` are assigned at every audio frame. Let's take a look again at the `render()` loop (the Bela code for this example can be found in (in `bela-code/potentiometers/render.cpp`).\n", + "\n", + "```cpp\n", + "void render(BelaContext *context, void *userData)\n", + "{\n", + "\tfor(unsigned int n = 0; n < context->audioFrames; n++) {\n", + "\t\tif(gAudioFramesPerAnalogFrame && !(n % gAudioFramesPerAnalogFrame)) {\n", + "\t\t\t\n", + "\t\t\tuint64_t frames = context->audioFramesElapsed/gAudioFramesPerAnalogFrame + n/gAudioFramesPerAnalogFrame;\n", + "\t\t\tBela_getDefaultWatcherManager()->tick(frames); // watcher timestamps\n", + "\t\t\t\n", + "\t\t\tpot1 = analogRead(context, n/gAudioFramesPerAnalogFrame, gPot1Ch);\n", + "\t\t\tpot2 = analogRead(context, n/gAudioFramesPerAnalogFrame, gPot2Ch);\n", + "\t\t\t\n", + "\t\t}\n", + "\t}\n", + "}\n", + "```\n", + "\n", + "\n", + "The Watched clock is also \"ticked\" at every analog frame, so that the timestamps in the data correspond to the audio frames in the Bela code. The data buffers we received from Bela in the Streamer and the Logger had this form: `{\"ref_timestamp\": 92381, \"data\":[0.34, 0.45, ...]}`. Each data point is registered in the buffer every time we assign a value to `pot1` and `pot2` in the Bela code. The `ref_timestamp` corresponds to the timestamp of the first sample in the `data` array, in this case `0.34`. Since in the Bela code, we assign `pot1` and `pot2` at every audio frame, we can infer the timestamps of each value in the data array by incrementing `ref_timestamp` by 1 for each sample. \n", + "\n", + "This is an efficient way of storing data since instead of storing the timestamp of every item in the data array, we only store the timestamp of the first item. We call this *dense* timestamping. However, for many applications, we might not assign a value to a variable every frame, we might do it more than once per frame, once every few frames, or we might want to do it at irregular intervals. In these cases, we need to store the timestamp of every item in the data array. We call this *sparse* timestamping.\n", + "\n", + "In this tutorial we take a look at *sparse* timestamping. The complete documentation for the pybela library can be found in [https://belaplatform.github.io/pybela/](https://belaplatform.github.io/pybela/).\n", + "\n", + "First, transfer the Bela code we will use in this tutorial to Bela:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!rsync -rvL ../bela-code/timestamping root@bela.local:Bela/projects" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then you can compile and run the project using either the IDE or by running the following command in the Terminal:\n", + "```bash\n", + "ssh root@bela.local \"make -C Bela stop Bela PROJECT=potentiometers run\" \n", + "```\n", + "(Running this on a jupyter notebook will block the cell until the program is stopped on Bela.)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As in the previous tutorials, we will use two potentiometers connected to Bela analog inputs 0 and 1. Check the `1_Streamer.ipnyb` tutorial notebook for instructions on how to set up the circuit. \n", + "\n", + "### Bela C++ code\n", + "\n", + "\n", + "First, let's take a look at the Bela code. First, we have added `WatcherManager::kTimestampSample` to the declaration of `pot2`. This informs the Bela Watcher that `pot2` will be watched sparsely, that is, that the watcher will store a timestamp for every value assigned to `pot2`:\n", + "\n", + "```cpp\n", + "Watcher pot1(\"pot1\");\n", + "Watcher pot2(\"pot2\", WatcherManager::kTimestampSample);\n", + "```\n", + "\n", + "Now let's take a look at `render()`:\n", + "\n", + "```cpp\n", + "void render(BelaContext *context, void *userData)\n", + "{\n", + "\tfor(unsigned int n = 0; n < context->audioFrames; n++) {\n", + "\t\tif(gAudioFramesPerAnalogFrame && !(n % gAudioFramesPerAnalogFrame)) {\n", + "\t\t\t\n", + "\t\t\tuint64_t frames = context->audioFramesElapsed/gAudioFramesPerAnalogFrame + n/gAudioFramesPerAnalogFrame;\n", + "\t\t\tBela_getDefaultWatcherManager()->tick(frames); // watcher timestamps\n", + "\t\t\t\n", + "\t\t\tpot1 = analogRead(context, n/gAudioFramesPerAnalogFrame, gPot1Ch);\n", + "\n", + "\t\t\tif (frames % 12==0){\n", + "\t\t\t\tpot2 = analogRead(context, n/gAudioFramesPerAnalogFrame, gPot2Ch);\n", + "\t\t\t}\n", + "\t\t}\n", + "\t}\n", + "}\n", + "```\n", + "\n", + "We are \"ticking\" the Bela Watcher once per analog frame, so that the timestamps in the data correspond to the analog frames in the Bela code. We are assigning a value to `pot1` at every analog frame, as in the previous examples, but we are now only assigning a value to `pot2` every 12 frames. \n", + "\n", + "### Dealing with sparse timestamps in Python\n", + "\n", + "Let's now take a look at the data we receive from Bela. We will use the Streamer. Run the cells below to declare and connect the Streamer to Bela:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from pybela import Streamer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer = Streamer()\n", + "streamer.connect()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can call `.list()` to take a look at the variables available to be streamed, their types and timestamp mode:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer.list()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`timestampMode` indicates if the timestamping is *sparse* (1) or *dense* (0). Now let's stream the data from Bela. We will stream `pot1` and `pot2`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer.start_streaming(variables=[\"pot1\", \"pot2\"], saving_enabled=False)\n", + "streamer.wait(2)\n", + "streamer.stop_streaming()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's take a look at the streamed buffers for \"pot2\". Each buffer has the form `{\"ref_timestamp\": 912831, \"data\":[0.23, 0.24, ...], \"rel_timestamps\":[ 0, 12, ...]}`. `ref_timestamp` corresponds, as in the dense case, to the timestamp of the first data point in the `data` array. `rel_timestamps` is an array of timestamps relative to `ref_timestamp`. In this case, since we are assigning a value to `pot2` every 12 frames, the timestamps in `rel_timestamps` are `[0, 12, 24, 36, etc.]`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer.streaming_buffers_queue[\"pot2\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can now calculate the absolute timestamps of each data point by adding the values in `rel_timestamps` to `ref_timestamp`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "[streamer.streaming_buffers_queue[\"pot2\"][0][\"ref_timestamp\"]]*len(streamer.streaming_buffers_queue[\"pot2\"][0][\"rel_timestamps\"]) + streamer.streaming_buffers_queue[\"pot2\"][0][\"rel_timestamps\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pot2_data = {\"timestamps\":[], \"data\":[]}\n", + "\n", + "for _buffer in streamer.streaming_buffers_queue[\"pot2\"]:\n", + " pot2_data[\"timestamps\"].extend([_buffer[\"ref_timestamp\"] + i for i in _buffer[\"rel_timestamps\"]])\n", + " pot2_data[\"data\"].extend(_buffer[\"data\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that the timestamps are spaced by 12, as expected:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(pot2_data)\n", + "df.head()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pybela-2uXYSGIe", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.19" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} From ad5aca59ada550c5b9c573b6d0b3cfa669bd372a Mon Sep 17 00:00:00 2001 From: Teresa Pelinski Date: Mon, 16 Dec 2024 21:53:40 +0000 Subject: [PATCH 5/5] removed loop=self.loop, fixed sphinx versions, paackaged v2.0.0 --- Pipfile | 5 +- Pipfile.lock | 308 +++++++++++++++++++++--------------- dev/dev.prepare-requires.py | 4 +- dev/dev.test-dist.sh | 2 +- pybela/Logger.py | 4 +- pybela/Streamer.py | 13 +- pybela/Watcher.py | 9 +- readme.md | 3 +- requirements.txt | 1 + setup.py | 2 +- 10 files changed, 206 insertions(+), 145 deletions(-) diff --git a/Pipfile b/Pipfile index 940eedd..f9fa8a0 100644 --- a/Pipfile +++ b/Pipfile @@ -17,12 +17,13 @@ numpy = "==1.26.0" bokeh = "==2.4.3" panel = "==0.14.4" jupyter-bokeh = "==3.0.5" +pandas = "==2.2.3" [dev-packages] twine = "*" pip-chill = "*" -sphinx = "*" -sphinx-rtd-theme = "*" +sphinx = "==7.4.7" +sphinx-rtd-theme = "==3.0.2" build = "*" pipdeptree = "*" diff --git a/Pipfile.lock b/Pipfile.lock index b18d530..9d9559c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4cd4f9cccc875ad6100077153e7e98a14ebb9a803222e0b8c261e512e36576e4" + "sha256": "240fc2c2226210a626f6a9fad32bb0393fecbec6bededac8708c1be40c0ba246" }, "pipfile-spec": 6, "requires": { @@ -26,11 +26,11 @@ }, "anyio": { "hashes": [ - "sha256:4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c", - "sha256:6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d" + "sha256:2f834749c602966b7d456a7567cafcb309f96482b5081d14ac93ccd457f9dd48", + "sha256:ea60c3723ab42ba6fff7e8ccb0488c898ec538ff4df1f1d5e642c3601d07e352" ], "markers": "python_version >= '3.9'", - "version": "==4.6.2.post1" + "version": "==4.7.0" }, "appnope": { "hashes": [ @@ -101,11 +101,11 @@ }, "attrs": { "hashes": [ - "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", - "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2" + "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff", + "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308" ], - "markers": "python_version >= '3.7'", - "version": "==24.2.0" + "markers": "python_version >= '3.8'", + "version": "==24.3.0" }, "babel": { "hashes": [ @@ -315,11 +315,11 @@ }, "certifi": { "hashes": [ - "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", - "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9" + "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56", + "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db" ], "markers": "python_version >= '3.6'", - "version": "==2024.8.30" + "version": "==2024.12.14" }, "cffi": { "hashes": [ @@ -550,35 +550,35 @@ }, "debugpy": { "hashes": [ - "sha256:1339e14c7d980407248f09824d1b25ff5c5616651689f1e0f0e51bdead3ea13e", - "sha256:17c5e0297678442511cf00a745c9709e928ea4ca263d764e90d233208889a19e", - "sha256:1efbb3ff61487e2c16b3e033bc8595aea578222c08aaf3c4bf0f93fadbd662ee", - "sha256:365e556a4772d7d0d151d7eb0e77ec4db03bcd95f26b67b15742b88cacff88e9", - "sha256:3d9755e77a2d680ce3d2c5394a444cf42be4a592caaf246dbfbdd100ffcf7ae5", - "sha256:3e59842d6c4569c65ceb3751075ff8d7e6a6ada209ceca6308c9bde932bcef11", - "sha256:472a3994999fe6c0756945ffa359e9e7e2d690fb55d251639d07208dbc37caea", - "sha256:54a7e6d3014c408eb37b0b06021366ee985f1539e12fe49ca2ee0d392d9ceca5", - "sha256:5e565fc54b680292b418bb809f1386f17081d1346dca9a871bf69a8ac4071afe", - "sha256:62d22dacdb0e296966d7d74a7141aaab4bec123fa43d1a35ddcb39bf9fd29d70", - "sha256:66eeae42f3137eb428ea3a86d4a55f28da9bd5a4a3d369ba95ecc3a92c1bba53", - "sha256:6953b335b804a41f16a192fa2e7851bdcfd92173cbb2f9f777bb934f49baab65", - "sha256:7c4d65d03bee875bcb211c76c1d8f10f600c305dbd734beaed4077e902606fee", - "sha256:7e646e62d4602bb8956db88b1e72fe63172148c1e25c041e03b103a25f36673c", - "sha256:7e8b079323a56f719977fde9d8115590cb5e7a1cba2fcee0986ef8817116e7c1", - "sha256:8138efff315cd09b8dcd14226a21afda4ca582284bf4215126d87342bba1cc66", - "sha256:8e99c0b1cc7bf86d83fb95d5ccdc4ad0586d4432d489d1f54e4055bcc795f693", - "sha256:957363d9a7a6612a37458d9a15e72d03a635047f946e5fceee74b50d52a9c8e2", - "sha256:957ecffff80d47cafa9b6545de9e016ae8c9547c98a538ee96ab5947115fb3dd", - "sha256:ada7fb65102a4d2c9ab62e8908e9e9f12aed9d76ef44880367bc9308ebe49a0f", - "sha256:b74a49753e21e33e7cf030883a92fa607bddc4ede1aa4145172debc637780040", - "sha256:c36856343cbaa448171cba62a721531e10e7ffb0abff838004701454149bc037", - "sha256:cc37a6c9987ad743d9c3a14fa1b1a14b7e4e6041f9dd0c8abf8895fe7a97b899", - "sha256:cfe1e6c6ad7178265f74981edf1154ffce97b69005212fbc90ca22ddfe3d017e", - "sha256:e46b420dc1bea64e5bbedd678148be512442bc589b0111bd799367cde051e71a", - "sha256:ff54ef77ad9f5c425398efb150239f6fe8e20c53ae2f68367eba7ece1e96226d" + "sha256:0e22f846f4211383e6a416d04b4c13ed174d24cc5d43f5fd52e7821d0ebc8920", + "sha256:116bf8342062246ca749013df4f6ea106f23bc159305843491f64672a55af2e5", + "sha256:189058d03a40103a57144752652b3ab08ff02b7595d0ce1f651b9acc3a3a35a0", + "sha256:23dc34c5e03b0212fa3c49a874df2b8b1b8fda95160bd79c01eb3ab51ea8d851", + "sha256:28e45b3f827d3bf2592f3cf7ae63282e859f3259db44ed2b129093ca0ac7940b", + "sha256:2b26fefc4e31ff85593d68b9022e35e8925714a10ab4858fb1b577a8a48cb8cd", + "sha256:32db46ba45849daed7ccf3f2e26f7a386867b077f39b2a974bb5c4c2c3b0a280", + "sha256:40499a9979c55f72f4eb2fc38695419546b62594f8af194b879d2a18439c97a9", + "sha256:44b1b8e6253bceada11f714acf4309ffb98bfa9ac55e4fce14f9e5d4484287a1", + "sha256:52c3cf9ecda273a19cc092961ee34eb9ba8687d67ba34cc7b79a521c1c64c4c0", + "sha256:52d8a3166c9f2815bfae05f386114b0b2d274456980d41f320299a8d9a5615a7", + "sha256:61bc8b3b265e6949855300e84dc93d02d7a3a637f2aec6d382afd4ceb9120c9f", + "sha256:654130ca6ad5de73d978057eaf9e582244ff72d4574b3e106fb8d3d2a0d32458", + "sha256:6ad2688b69235c43b020e04fecccdf6a96c8943ca9c2fb340b8adc103c655e57", + "sha256:6c1f6a173d1140e557347419767d2b14ac1c9cd847e0b4c5444c7f3144697e4e", + "sha256:84e511a7545d11683d32cdb8f809ef63fc17ea2a00455cc62d0a4dbb4ed1c308", + "sha256:85de8474ad53ad546ff1c7c7c89230db215b9b8a02754d41cb5a76f70d0be296", + "sha256:8988f7163e4381b0da7696f37eec7aca19deb02e500245df68a7159739bbd0d3", + "sha256:8da1db4ca4f22583e834dcabdc7832e56fe16275253ee53ba66627b86e304da1", + "sha256:8ffc382e4afa4aee367bf413f55ed17bd91b191dcaf979890af239dda435f2a1", + "sha256:987bce16e86efa86f747d5151c54e91b3c1e36acc03ce1ddb50f9d09d16ded0e", + "sha256:ad7efe588c8f5cf940f40c3de0cd683cc5b76819446abaa50dc0829a30c094db", + "sha256:bb3b15e25891f38da3ca0740271e63ab9db61f41d4d8541745cfc1824252cb28", + "sha256:c928bbf47f65288574b78518449edaa46c82572d340e2750889bbf8cd92f3737", + "sha256:ce291a5aca4985d82875d6779f61375e959208cdf09fcec40001e65fb0a54768", + "sha256:d8768edcbeb34da9e11bcb8b5c2e0958d25218df7a6e56adf415ef262cd7b6d1" ], "markers": "python_version >= '3.8'", - "version": "==1.8.9" + "version": "==1.8.11" }, "decorator": { "hashes": [ @@ -644,11 +644,11 @@ }, "httpx": { "hashes": [ - "sha256:0858d3bab51ba7e386637f22a61d8ccddaeec5f3fe4209da3a6168dbb91573e0", - "sha256:dc0b419a0cfeb6e8b34e85167c0da2671206f5095f1baa9663d23bcfd6b535fc" + "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", + "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad" ], "markers": "python_version >= '3.8'", - "version": "==0.28.0" + "version": "==0.28.1" }, "idna": { "hashes": [ @@ -1040,6 +1040,54 @@ "markers": "python_version >= '3.8'", "version": "==24.2" }, + "pandas": { + "hashes": [ + "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a", + "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d", + "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5", + "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4", + "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0", + "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32", + "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea", + "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28", + "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f", + "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348", + "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18", + "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468", + "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5", + "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e", + "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", + "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645", + "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13", + "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30", + "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3", + "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d", + "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb", + "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3", + "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039", + "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8", + "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd", + "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761", + "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659", + "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57", + "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c", + "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c", + "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4", + "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a", + "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9", + "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42", + "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2", + "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39", + "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc", + "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698", + "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed", + "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015", + "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24", + "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319" + ], + "index": "pypi", + "version": "==2.2.3" + }, "pandocfilters": { "hashes": [ "sha256:002b4a555ee4ebc03f8b66307e287fa492e4a77b4ea14d3f934328297bb4939e", @@ -1285,11 +1333,18 @@ }, "python-json-logger": { "hashes": [ - "sha256:23e7ec02d34237c5aa1e29a070193a4ea87583bb4e7f8fd06d3de8264c4b2e1c", - "sha256:f380b826a991ebbe3de4d897aeec42760035ac760345e57b812938dc8b35e2bd" + "sha256:8eb0554ea17cb75b05d2848bc14fb02fbdbd9d6972120781b974380bfa162008", + "sha256:cdc17047eb5374bd311e748b42f99d71223f3b0e186f4206cc5d52aefe85b090" ], - "markers": "python_version >= '3.6'", - "version": "==2.0.7" + "markers": "python_version >= '3.8'", + "version": "==3.2.1" + }, + "pytz": { + "hashes": [ + "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a", + "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725" + ], + "version": "==2024.2" }, "pyviz-comms": { "hashes": [ @@ -1750,11 +1805,11 @@ }, "types-python-dateutil": { "hashes": [ - "sha256:250e1d8e80e7bbc3a6c99b907762711d1a1cdd00e978ad39cb5940f6f0a87f3d", - "sha256:58cb85449b2a56d6684e41aeefb4c4280631246a0da1a719bdbe6f3fb0317446" + "sha256:18f493414c26ffba692a72369fea7a154c502646301ebfe3d56a04b3767284cb", + "sha256:e248a4bc70a486d3e3ec84d0dc30eec3a5f979d6e7ee4123ae043eedbb987f53" ], "markers": "python_version >= '3.8'", - "version": "==2.9.0.20241003" + "version": "==2.9.0.20241206" }, "typing-extensions": { "hashes": [ @@ -1764,6 +1819,14 @@ "markers": "python_version >= '3.8'", "version": "==4.12.2" }, + "tzdata": { + "hashes": [ + "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc", + "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd" + ], + "markers": "python_version >= '2'", + "version": "==2024.2" + }, "uri-template": { "hashes": [ "sha256:0e00f8eb65e18c7de20d595a14336e9f337ead580c70934141624b6d1ffdacc7", @@ -1810,81 +1873,78 @@ }, "websockets": { "hashes": [ - "sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b", - "sha256:0bee75f400895aef54157b36ed6d3b308fcab62e5260703add87f44cee9c82a6", - "sha256:0e6e2711d5a8e6e482cacb927a49a3d432345dfe7dea8ace7b5790df5932e4df", - "sha256:12743ab88ab2af1d17dd4acb4645677cb7063ef4db93abffbf164218a5d54c6b", - "sha256:1a9d160fd080c6285e202327aba140fc9a0d910b09e423afff4ae5cbbf1c7205", - "sha256:1bf386089178ea69d720f8db6199a0504a406209a0fc23e603b27b300fdd6892", - "sha256:1df2fbd2c8a98d38a66f5238484405b8d1d16f929bb7a33ed73e4801222a6f53", - "sha256:1e4b3f8ea6a9cfa8be8484c9221ec0257508e3a1ec43c36acdefb2a9c3b00aa2", - "sha256:1f38a7b376117ef7aff996e737583172bdf535932c9ca021746573bce40165ed", - "sha256:23509452b3bc38e3a057382c2e941d5ac2e01e251acce7adc74011d7d8de434c", - "sha256:248d8e2446e13c1d4326e0a6a4e9629cb13a11195051a73acf414812700badbd", - "sha256:25eb766c8ad27da0f79420b2af4b85d29914ba0edf69f547cc4f06ca6f1d403b", - "sha256:27a5e9964ef509016759f2ef3f2c1e13f403725a5e6a1775555994966a66e931", - "sha256:2c71bd45a777433dd9113847af751aae36e448bc6b8c361a566cb043eda6ec30", - "sha256:2cb388a5bfb56df4d9a406783b7f9dbefb888c09b71629351cc6b036e9259370", - "sha256:2d225bb6886591b1746b17c0573e29804619c8f755b5598d875bb4235ea639be", - "sha256:2e5fc14ec6ea568200ea4ef46545073da81900a2b67b3e666f04adf53ad452ec", - "sha256:363f57ca8bc8576195d0540c648aa58ac18cf85b76ad5202b9f976918f4219cf", - "sha256:3c6cc1360c10c17463aadd29dd3af332d4a1adaa8796f6b0e9f9df1fdb0bad62", - "sha256:3d829f975fc2e527a3ef2f9c8f25e553eb7bc779c6665e8e1d52aa22800bb38b", - "sha256:3e3aa8c468af01d70332a382350ee95f6986db479ce7af14d5e81ec52aa2b402", - "sha256:3f61726cae9f65b872502ff3c1496abc93ffbe31b278455c418492016e2afc8f", - "sha256:423fc1ed29f7512fceb727e2d2aecb952c46aa34895e9ed96071821309951123", - "sha256:46e71dbbd12850224243f5d2aeec90f0aaa0f2dde5aeeb8fc8df21e04d99eff9", - "sha256:4d87be612cbef86f994178d5186add3d94e9f31cc3cb499a0482b866ec477603", - "sha256:5693ef74233122f8ebab026817b1b37fe25c411ecfca084b29bc7d6efc548f45", - "sha256:5aa9348186d79a5f232115ed3fa9020eab66d6c3437d72f9d2c8ac0c6858c558", - "sha256:5d873c7de42dea355d73f170be0f23788cf3fa9f7bed718fd2830eefedce01b4", - "sha256:5f6ffe2c6598f7f7207eef9a1228b6f5c818f9f4d53ee920aacd35cec8110438", - "sha256:604428d1b87edbf02b233e2c207d7d528460fa978f9e391bd8aaf9c8311de137", - "sha256:6350b14a40c95ddd53e775dbdbbbc59b124a5c8ecd6fbb09c2e52029f7a9f480", - "sha256:6e2df67b8014767d0f785baa98393725739287684b9f8d8a1001eb2839031447", - "sha256:6e96f5ed1b83a8ddb07909b45bd94833b0710f738115751cdaa9da1fb0cb66e8", - "sha256:6e9e7db18b4539a29cc5ad8c8b252738a30e2b13f033c2d6e9d0549b45841c04", - "sha256:70ec754cc2a769bcd218ed8d7209055667b30860ffecb8633a834dde27d6307c", - "sha256:7b645f491f3c48d3f8a00d1fce07445fab7347fec54a3e65f0725d730d5b99cb", - "sha256:7fa3d25e81bfe6a89718e9791128398a50dec6d57faf23770787ff441d851967", - "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b", - "sha256:8572132c7be52632201a35f5e08348137f658e5ffd21f51f94572ca6c05ea81d", - "sha256:87b4aafed34653e465eb77b7c93ef058516cb5acf3eb21e42f33928616172def", - "sha256:8e332c210b14b57904869ca9f9bf4ca32f5427a03eeb625da9b616c85a3a506c", - "sha256:9893d1aa45a7f8b3bc4510f6ccf8db8c3b62120917af15e3de247f0780294b92", - "sha256:9edf3fc590cc2ec20dc9d7a45108b5bbaf21c0d89f9fd3fd1685e223771dc0b2", - "sha256:9fdf06fd06c32205a07e47328ab49c40fc1407cdec801d698a7c41167ea45113", - "sha256:a02413bc474feda2849c59ed2dfb2cddb4cd3d2f03a2fedec51d6e959d9b608b", - "sha256:a1d9697f3337a89691e3bd8dc56dea45a6f6d975f92e7d5f773bc715c15dde28", - "sha256:a571f035a47212288e3b3519944f6bf4ac7bc7553243e41eac50dd48552b6df7", - "sha256:ab3d732ad50a4fbd04a4490ef08acd0517b6ae6b77eb967251f4c263011a990d", - "sha256:ae0a5da8f35a5be197f328d4727dbcfafa53d1824fac3d96cdd3a642fe09394f", - "sha256:b067cb952ce8bf40115f6c19f478dc71c5e719b7fbaa511359795dfd9d1a6468", - "sha256:b2ee7288b85959797970114deae81ab41b731f19ebcd3bd499ae9ca0e3f1d2c8", - "sha256:b81f90dcc6c85a9b7f29873beb56c94c85d6f0dac2ea8b60d995bd18bf3e2aae", - "sha256:ba0cab91b3956dfa9f512147860783a1829a8d905ee218a9837c18f683239611", - "sha256:baa386875b70cbd81798fa9f71be689c1bf484f65fd6fb08d051a0ee4e79924d", - "sha256:bbe6013f9f791944ed31ca08b077e26249309639313fff132bfbf3ba105673b9", - "sha256:bea88d71630c5900690fcb03161ab18f8f244805c59e2e0dc4ffadae0a7ee0ca", - "sha256:befe90632d66caaf72e8b2ed4d7f02b348913813c8b0a32fae1cc5fe3730902f", - "sha256:c3181df4583c4d3994d31fb235dc681d2aaad744fbdbf94c4802485ececdecf2", - "sha256:c4e37d36f0d19f0a4413d3e18c0d03d0c268ada2061868c1e6f5ab1a6d575077", - "sha256:c588f6abc13f78a67044c6b1273a99e1cf31038ad51815b3b016ce699f0d75c2", - "sha256:cbe83a6bbdf207ff0541de01e11904827540aa069293696dd528a6640bd6a5f6", - "sha256:d554236b2a2006e0ce16315c16eaa0d628dab009c33b63ea03f41c6107958374", - "sha256:dbcf72a37f0b3316e993e13ecf32f10c0e1259c28ffd0a85cee26e8549595fbc", - "sha256:dc284bbc8d7c78a6c69e0c7325ab46ee5e40bb4d50e494d8131a07ef47500e9e", - "sha256:dff6cdf35e31d1315790149fee351f9e52978130cef6c87c4b6c9b3baf78bc53", - "sha256:e469d01137942849cff40517c97a30a93ae79917752b34029f0ec72df6b46399", - "sha256:eb809e816916a3b210bed3c82fb88eaf16e8afcf9c115ebb2bacede1797d2547", - "sha256:ed2fcf7a07334c77fc8a230755c2209223a7cc44fc27597729b8ef5425aa61a3", - "sha256:f44069528d45a933997a6fef143030d8ca8042f0dfaad753e2906398290e2870", - "sha256:f764ba54e33daf20e167915edc443b6f88956f37fb606449b4a5b10ba42235a5", - "sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8", - "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7" + "sha256:00fe5da3f037041da1ee0cf8e308374e236883f9842c7c465aa65098b1c9af59", + "sha256:01bb2d4f0a6d04538d3c5dfd27c0643269656c28045a53439cbf1c004f90897a", + "sha256:034feb9f4286476f273b9a245fb15f02c34d9586a5bc936aff108c3ba1b21beb", + "sha256:04a97aca96ca2acedf0d1f332c861c5a4486fdcba7bcef35873820f940c4231e", + "sha256:0d4290d559d68288da9f444089fd82490c8d2744309113fc26e2da6e48b65da6", + "sha256:1288369a6a84e81b90da5dbed48610cd7e5d60af62df9851ed1d1d23a9069f10", + "sha256:14839f54786987ccd9d03ed7f334baec0f02272e7ec4f6e9d427ff584aeea8b4", + "sha256:1d045cbe1358d76b24d5e20e7b1878efe578d9897a25c24e6006eef788c0fdf0", + "sha256:1f874ba705deea77bcf64a9da42c1f5fc2466d8f14daf410bc7d4ceae0a9fcb0", + "sha256:205f672a6c2c671a86d33f6d47c9b35781a998728d2c7c2a3e1cf3333fcb62b7", + "sha256:2177ee3901075167f01c5e335a6685e71b162a54a89a56001f1c3e9e3d2ad250", + "sha256:219c8187b3ceeadbf2afcf0f25a4918d02da7b944d703b97d12fb01510869078", + "sha256:25225cc79cfebc95ba1d24cd3ab86aaa35bcd315d12fa4358939bd55e9bd74a5", + "sha256:3630b670d5057cd9e08b9c4dab6493670e8e762a24c2c94ef312783870736ab9", + "sha256:368a05465f49c5949e27afd6fbe0a77ce53082185bbb2ac096a3a8afaf4de52e", + "sha256:36ebd71db3b89e1f7b1a5deaa341a654852c3518ea7a8ddfdf69cc66acc2db1b", + "sha256:39450e6215f7d9f6f7bc2a6da21d79374729f5d052333da4d5825af8a97e6735", + "sha256:398b10c77d471c0aab20a845e7a60076b6390bfdaac7a6d2edb0d2c59d75e8d8", + "sha256:3c3deac3748ec73ef24fc7be0b68220d14d47d6647d2f85b2771cb35ea847aa1", + "sha256:3f14a96a0034a27f9d47fd9788913924c89612225878f8078bb9d55f859272b0", + "sha256:3fc753451d471cff90b8f467a1fc0ae64031cf2d81b7b34e1811b7e2691bc4bc", + "sha256:414ffe86f4d6f434a8c3b7913655a1a5383b617f9bf38720e7c0799fac3ab1c6", + "sha256:449d77d636f8d9c17952628cc7e3b8faf6e92a17ec581ec0c0256300717e1512", + "sha256:4b6caec8576e760f2c7dd878ba817653144d5f369200b6ddf9771d64385b84d4", + "sha256:4d4fc827a20abe6d544a119896f6b78ee13fe81cbfef416f3f2ddf09a03f0e2e", + "sha256:5a42d3ecbb2db5080fc578314439b1d79eef71d323dc661aa616fb492436af5d", + "sha256:5b918d288958dc3fa1c5a0b9aa3256cb2b2b84c54407f4813c45d52267600cd3", + "sha256:5ef440054124728cc49b01c33469de06755e5a7a4e83ef61934ad95fc327fbb0", + "sha256:660c308dabd2b380807ab64b62985eaccf923a78ebc572bd485375b9ca2b7dc7", + "sha256:6a6c9bcf7cdc0fd41cc7b7944447982e8acfd9f0d560ea6d6845428ed0562058", + "sha256:6d24fc337fc055c9e83414c94e1ee0dee902a486d19d2a7f0929e49d7d604b09", + "sha256:7048eb4415d46368ef29d32133134c513f507fff7d953c18c91104738a68c3b3", + "sha256:77569d19a13015e840b81550922056acabc25e3f52782625bc6843cfa034e1da", + "sha256:8149a0f5a72ca36720981418eeffeb5c2729ea55fa179091c81a0910a114a5d2", + "sha256:836bef7ae338a072e9d1863502026f01b14027250a4545672673057997d5c05a", + "sha256:8621a07991add373c3c5c2cf89e1d277e49dc82ed72c75e3afc74bd0acc446f0", + "sha256:87e31011b5c14a33b29f17eb48932e63e1dcd3fa31d72209848652310d3d1f0d", + "sha256:88cf9163ef674b5be5736a584c999e98daf3aabac6e536e43286eb74c126b9c7", + "sha256:8fda642151d5affdee8a430bd85496f2e2517be3a2b9d2484d633d5712b15c56", + "sha256:90b5d9dfbb6d07a84ed3e696012610b6da074d97453bd01e0e30744b472c8179", + "sha256:90f4c7a069c733d95c308380aae314f2cb45bd8a904fb03eb36d1a4983a4993f", + "sha256:9481a6de29105d73cf4515f2bef8eb71e17ac184c19d0b9918a3701c6c9c4f23", + "sha256:9607b9a442392e690a57909c362811184ea429585a71061cd5d3c2b98065c199", + "sha256:9777564c0a72a1d457f0848977a1cbe15cfa75fa2f67ce267441e465717dcf1a", + "sha256:a032855dc7db987dff813583d04f4950d14326665d7e714d584560b140ae6b8b", + "sha256:a0adf84bc2e7c86e8a202537b4fd50e6f7f0e4a6b6bf64d7ccb96c4cd3330b29", + "sha256:a35f704be14768cea9790d921c2c1cc4fc52700410b1c10948511039be824aac", + "sha256:a3dfff83ca578cada2d19e665e9c8368e1598d4e787422a460ec70e531dbdd58", + "sha256:a4c805c6034206143fbabd2d259ec5e757f8b29d0a2f0bf3d2fe5d1f60147a4a", + "sha256:a655bde548ca98f55b43711b0ceefd2a88a71af6350b0c168aa77562104f3f45", + "sha256:ad2ab2547761d79926effe63de21479dfaf29834c50f98c4bf5b5480b5838434", + "sha256:b1f3628a0510bd58968c0f60447e7a692933589b791a6b572fcef374053ca280", + "sha256:b7e7ea2f782408c32d86b87a0d2c1fd8871b0399dd762364c731d86c86069a78", + "sha256:bc6ccf7d54c02ae47a48ddf9414c54d48af9c01076a2e1023e3b486b6e72c707", + "sha256:bea45f19b7ca000380fbd4e02552be86343080120d074b87f25593ce1700ad58", + "sha256:cc1fc87428c1d18b643479caa7b15db7d544652e5bf610513d4a3478dbe823d0", + "sha256:cd7c11968bc3860d5c78577f0dbc535257ccec41750675d58d8dc66aa47fe52c", + "sha256:ceada5be22fa5a5a4cdeec74e761c2ee7db287208f54c718f2df4b7e200b8d4a", + "sha256:cf5201a04550136ef870aa60ad3d29d2a59e452a7f96b94193bee6d73b8ad9a9", + "sha256:d9fd19ecc3a4d5ae82ddbfb30962cf6d874ff943e56e0c81f5169be2fda62979", + "sha256:ddaa4a390af911da6f680be8be4ff5aaf31c4c834c1a9147bc21cbcbca2d4370", + "sha256:df174ece723b228d3e8734a6f2a6febbd413ddec39b3dc592f5a4aa0aff28098", + "sha256:e0744623852f1497d825a49a99bfbec9bea4f3f946df6eb9d8a2f0c37a2fec2e", + "sha256:e5dc25a9dbd1a7f61eca4b7cb04e74ae4b963d658f9e4f9aad9cd00b688692c8", + "sha256:e7591d6f440af7f73c4bd9404f3772bfee064e639d2b6cc8c94076e71b2471c1", + "sha256:eb6d38971c800ff02e4a6afd791bbe3b923a9a57ca9aeab7314c21c84bf9ff05", + "sha256:ed907449fe5e021933e46a3e65d651f641975a768d0649fee59f10c2985529ed", + "sha256:f6cf0ad281c979306a6a34242b371e90e891bce504509fb6bb5246bbbf31e7b6", + "sha256:f95ba34d71e2fa0c5d225bde3b3bdb152e957150100e75c86bc7f3964c450d89" ], "index": "pypi", - "version": "==12.0" + "version": "==14.1" }, "widgetsnbextension": { "hashes": [ @@ -1938,11 +1998,11 @@ }, "certifi": { "hashes": [ - "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", - "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9" + "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56", + "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db" ], "markers": "python_version >= '3.6'", - "version": "==2024.8.30" + "version": "==2024.12.14" }, "charset-normalizer": { "hashes": [ diff --git a/dev/dev.prepare-requires.py b/dev/dev.prepare-requires.py index 10cac22..1d9ae29 100644 --- a/dev/dev.prepare-requires.py +++ b/dev/dev.prepare-requires.py @@ -6,17 +6,17 @@ def filter_pybela_packages(file_path): for line in lines: if 'pybela' in line and line.startswith('#'): pybela_packages.append(line.split('#')[1].split(' ')[1]) - + print(pybela_packages) return pybela_packages def write_pybela_packages_to_file(pybela_packages, output_file_path): + print(f"Writing pybela packages to file: {output_file_path}") with open(output_file_path, 'w') as file: for package in pybela_packages: file.write(f"{package}\n") -# Example usage file_path = 'pip-chill.txt' output_file_path = 'requirements.txt' pybela_packages = filter_pybela_packages(file_path) diff --git a/dev/dev.test-dist.sh b/dev/dev.test-dist.sh index e917c9c..485a3e4 100644 --- a/dev/dev.test-dist.sh +++ b/dev/dev.test-dist.sh @@ -7,7 +7,7 @@ echo "\nCreating test-env..." python -m venv test-env source test-env/bin/activate echo "\nInstalling pybela from dist..." -pip install ../dist/pybela-1.0.2-py3-none-any.whl +pip install ../dist/pybela-2.0.0-py3-none-any.whl echo "\nRunning test.py..." python test.py deactivate diff --git a/pybela/Logger.py b/pybela/Logger.py index 1014559..4c13498 100644 --- a/pybela/Logger.py +++ b/pybela/Logger.py @@ -441,12 +441,12 @@ async def _async_copy_file_from_bela(self, remote_path, local_path, verbose=Fals _local_path = self._generate_local_filename(local_path) else: _local_path = local_path - transferred_event = asyncio.Event(loop=self.loop) + transferred_event = asyncio.Event() def callback(transferred, to_transfer): return transferred_event.set( ) if transferred == to_transfer else None self.sftp_client.get(remote_path, _local_path, callback=callback) file_size = self.sftp_client.stat(remote_path).st_size - await asyncio.wait_for(transferred_event.wait(), timeout=file_size/1e5) + await asyncio.wait_for(transferred_event.wait(), timeout=file_size*1e-4) if verbose: _print_ok( f"\rTransferring {remote_path}-->{_local_path}... Done.") diff --git a/pybela/Streamer.py b/pybela/Streamer.py index 4ef1052..7886d6c 100644 --- a/pybela/Streamer.py +++ b/pybela/Streamer.py @@ -36,14 +36,14 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add # -- streaming -- self._streaming_mode = "OFF" # OFF, FOREVER, N_VALUES, PEEK :: this flag prevents writing into the streaming buffer unless requested by the user using the start/stop_streaming() functions - self._streaming_buffer_available = asyncio.Event(loop=self.loop) + self._streaming_buffer_available = asyncio.Event() # number of streaming buffers (not of data points!) self._streaming_buffers_queue_length = 1000 self._streaming_buffers_queue = None self.last_streamed_buffer = {} # -- on data/block callbacks -- - self._processed_data_msg_queue = asyncio.Queue(loop=self.loop) + self._processed_data_msg_queue = asyncio.Queue() self._on_buffer_callback_is_active = False self._on_buffer_callback_worker_task = None self._on_block_callback_is_active = False @@ -59,7 +59,7 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add # -- monitor -- # stores the list of monitored variables for each monitored session. cleaned after each monitoring session. used to avoid calling list() every time a new message is parsed self._monitored_vars = None - self._peek_response_available = asyncio.Event(loop=self.loop) + self._peek_response_available = asyncio.Event() self._peek_response = None self._periods = None @@ -154,7 +154,7 @@ def __streaming_common_routine(self, variables=[], saving_enabled=False, saving_ self._streaming_buffers_queue = {var["name"]: deque( maxlen=self._streaming_buffers_queue_length) for var in self.watcher_vars} # clear asyncio data queues - self._processed_data_msg_queue = asyncio.Queue(loop=self.loop) + self._processed_data_msg_queue = asyncio.Queue() self.last_streamed_buffer = { var["name"]: {"data": [], "timestamps": []} for var in self.watcher_vars} @@ -251,7 +251,7 @@ async def _async_stop_streaming(self, variables=[]): self._saving_enabled = False self._saving_filename = None # await all active saving tasks - await asyncio.gather(*self._active_saving_tasks, return_exceptions=True, loop=self.loop) + await asyncio.gather(*self._active_saving_tasks, return_exceptions=True) self._active_saving_tasks.clear() _all_vars = [var["name"] for var in self.watcher_vars] @@ -270,8 +270,7 @@ async def _async_stop_streaming(self, variables=[]): if not _previous_streaming_mode == "PEEK": _print_info(f"Stopped monitoring variables {variables}...") - self._processed_data_msg_queue = asyncio.Queue( - loop=self.loop) # clear processed data queue + self._processed_data_msg_queue = asyncio.Queue() # clear processed data queue self._on_buffer_callback_is_active = False if self._on_buffer_callback_worker_task: self._on_buffer_callback_worker_task.cancel() diff --git a/pybela/Watcher.py b/pybela/Watcher.py index 6c05111..d44e594 100644 --- a/pybela/Watcher.py +++ b/pybela/Watcher.py @@ -51,6 +51,7 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add try: get_ipython().__class__.__name__ nest_asyncio.apply() + print("Running in Jupyter notebook. Enabling nest_asyncio.") except NameError: pass @@ -71,10 +72,10 @@ def __init__(self, ip="192.168.7.2", port=5555, data_add="gui_data", control_add self._send_ctrl_msg_task = None # queues - self._received_data_msg_queue = asyncio.Queue(loop=self.loop) - self._list_response_queue = asyncio.Queue(loop=self.loop) - self._to_send_data_msg_queue = asyncio.Queue(loop=self.loop) - self._to_send_ctrl_msg_queue = asyncio.Queue(loop=self.loop) + self._received_data_msg_queue = asyncio.Queue() + self._list_response_queue = asyncio.Queue() + self._to_send_data_msg_queue = asyncio.Queue() + self._to_send_ctrl_msg_queue = asyncio.Queue() # debug self._printall_responses = False diff --git a/readme.md b/readme.md index d46b272..81ebaf9 100644 --- a/readme.md +++ b/readme.md @@ -196,7 +196,6 @@ pipenv run python -m build --sdist # builds the .tar.gz file **Long term** -- [ ] **Design**: remove nest_asyncio? - [ ] **Add**: example projects - [ ] **Issue:** Monitor and streamer/controller can't be used simultaneously –  This is due to both monitor and streamer both using the same websocket connection and message format. This could be fixed by having a different message format for the monitor and the streamer (e.g., adding a header to the message) - [ ] **Issue:** The plotting routine does not work when variables are updated at different rates. @@ -204,7 +203,7 @@ pipenv run python -m build --sdist # builds the .tar.gz file - [ ] **Code refactor:** There are two routines for generating filenames (for Streamer and for Logger). This should be unified. - [ ] **Possible feature:** Flexible backend buffer size for streaming: if the assign rate of variables is too slow, the buffers might not be filled and hence not sent (since the data flushed is not collected in the frontend), and there will be long delays between the variable assign and the data being sent to the frontend. - [ ] **Issue:** Flushed buffers are not collected after `stop_streaming` in the frontend. -- [ ] **Bug:** `OSError: [Errno 12] Cannot allocate memory` + ## License diff --git a/requirements.txt b/requirements.txt index aa9d1f7..720564f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ numpy==1.26.0 panel==0.14.4 paramiko==3.5.0 websockets==14.1 +pandas==2.2.3 \ No newline at end of file diff --git a/setup.py b/setup.py index a8607bf..01c10dd 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ def load_requirements(filename): setuptools.setup( name="pybela", - version="1.0.2", + version="2.0.0", author="Teresa Pelinski", author_email="teresapelinski@gmail.com", description="pybela allows interfacing with Bela, the embedded audio platform, using python. It offers a convenient way to stream data between Bela and python, in both directions. In addition to data streaming, pybela supports data logging, as well as variable monitoring and control functionalities.",