From 2ac1e6a4e13f09ff024b2e94ed86230778aab324 Mon Sep 17 00:00:00 2001 From: Nathan Clack Date: Tue, 28 Nov 2023 10:57:54 -0800 Subject: [PATCH 01/12] (wip) --- Cargo.toml | 12 +++---- build.rs | 2 +- src/runtime.rs | 92 ++++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 82 insertions(+), 24 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 16bf190..34c803c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,24 +9,24 @@ name = "acquire" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.19", features = [ +pyo3 = { version = "0.20", features = [ "extension-module", "anyhow", "abi3-py38", "serde", ] } -pyo3-log = "0.8" -numpy = "0.19" +pyo3-log = "0.9" +numpy = "0.20" log = "0.4" anyhow = "1.0" parking_lot = "0.12" serde = { version = "1.0", features = ["derive"] } -pythonize = "0.19" +pythonize = "0.20.0" [build-dependencies] -bindgen = "0.66" +bindgen = "0.69.1" cmake = "0.1" -http = "0.2" +http = "1.0" json = "0.12" reqwest = { version = "0.11", features = ["blocking", "json"] } serde = { version = "1.0", features = ["derive"] } diff --git a/build.rs b/build.rs index 26ceed5..fbb6854 100644 --- a/build.rs +++ b/build.rs @@ -72,7 +72,7 @@ fn main() { let bindings = bindgen::Builder::default() .header("wrapper.h") .clang_arg(format!("-I{}/include", dst.display())) - .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) .generate() .expect("Unable to generate bindings"); diff --git a/src/runtime.rs b/src/runtime.rs index e33a77c..54f12db 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -77,6 +77,22 @@ impl RawRuntime { unsafe { capi::acquire_abort(self.inner.as_ptr()) }.ok()?; Ok(()) } + + fn map_read(&self, stream_id: u32) -> Result<(*mut capi::VideoFrame, *mut capi::VideoFrame)> { + let mut beg = null_mut(); + let mut end = null_mut(); + unsafe { + capi::acquire_map_read(self.inner.as_ptr(), stream_id, &mut beg, &mut end).ok()?; + } + Ok((beg, end)) + } + + fn unmap_read(&self, stream_id: u32, consumed_bytes: usize) -> Result<()> { + unsafe { + capi::acquire_unmap_read(self.inner.as_ptr(), stream_id, consumed_bytes).ok()?; + } + Ok(()) + } } impl Drop for RawRuntime { @@ -167,6 +183,7 @@ impl Runtime { } fn get_available_data(&self, stream_id: u32) -> PyResult> { + let (beg, end) = self.inner.map_read(stream_id); let mut beg = null_mut(); let mut end = null_mut(); unsafe { @@ -262,42 +279,83 @@ impl Drop for RawAvailableData { self.end.as_ptr(), consumed_bytes ); - unsafe { - capi::acquire_unmap_read( - self.runtime.inner.as_ptr(), - self.stream_id, - consumed_bytes as _, - ) - .ok() - .expect("Unexpected failure: Was the CoreRuntime NULL?"); - } + self.runtime + .unmap_read(self.stream_id, consumed_bytes) + .expect("Unexpected failure: Was the runtime NULL?"); } } #[pyclass] pub(crate) struct AvailableData { - inner: Arc, + inner: Option>, } #[pymethods] impl AvailableData { fn get_frame_count(&self) -> usize { - self.inner.get_frame_count() + self.inner.map(|x| x.get_frame_count()).unwrap_or(0) } - fn frames(&self) -> VideoFrameIterator { - VideoFrameIterator { - store: self.inner.clone(), - cur: Mutex::new(self.inner.beg), - end: self.inner.end, + fn frames(&self) -> Option { + if let Some(frames) = self.inner { + Some(VideoFrameIterator { + store: frames.clone(), + cur: Mutex::new(frames.beg), + end: frames.end, + }) + } else { + None } } fn __iter__(slf: PyRef<'_, Self>) -> PyResult> { - Py::new(slf.py(), slf.frames()) + if let Some(frames) = self.frames() { + Py::new(slf.py(), frames) + } else { + None + } } } +#[pyclass] +pub(crate) struct AvailableDataContext { + inner: Arc, + stream_id: u32, +} + +#[pymethods] +impl AvailableDataContext { + fn __enter__(&self) -> PyResult> { + let AvailableDataContext { inner, stream_id } = *self; + let (beg, end) = inner.map_read(stream_id)?; + let nbytes = unsafe { byte_offset_from(beg, end) }; + if nbytes > 0 { + log::trace!( + "[stream {}] ACQUIRED {:p}-{:p}:{} bytes", + stream_id, + beg, + end, + nbytes + ) + }; + Ok(if nbytes > 0 { + Some(AvailableData { + inner: Arc::new(RawAvailableData { + runtime: self.inner.clone(), + beg: NonNull::new(beg).ok_or(anyhow!("Expected non-null buffer"))?, + end: NonNull::new(end).ok_or(anyhow!("Expected non-null buffer"))?, + stream_id, + consumed_bytes: None, + }), + }) + } else { + None + }) + } + + fn __exit__(&self, _exc_type: PyAny, _exc_value: PyAny, _traceback: PyAny) {} +} + #[pyclass] struct VideoFrameIterator { store: Arc, From 0bbd173ae662200bd7b24ac9c2f1bc3d38570f65 Mon Sep 17 00:00:00 2001 From: Nathan Clack Date: Tue, 28 Nov 2023 17:24:36 -0800 Subject: [PATCH 02/12] (wip) testing test_repeat_acq --- python/acquire/acquire.pyi | 7 +- src/runtime.rs | 161 ++++++++++++++++++++----------------- tests/test_basic.py | 24 +++--- 3 files changed, 106 insertions(+), 86 deletions(-) diff --git a/python/acquire/acquire.pyi b/python/acquire/acquire.pyi index 33587a7..e584f72 100644 --- a/python/acquire/acquire.pyi +++ b/python/acquire/acquire.pyi @@ -12,6 +12,11 @@ from typing import ( from numpy.typing import NDArray +@final +class AvailableDataContext: + def __enter__(self)->Optional[AvailableData]: ... + def __exit__(self,exc_type:Any,exc_value:Any,traceback:Any)->None: ... + @final class AvailableData: def frames(self) -> Iterator[VideoFrame]: ... @@ -144,7 +149,7 @@ class Properties: class Runtime: def __init__(self, *args: None, **kwargs: Any) -> None: ... def device_manager(self) -> DeviceManager: ... - def get_available_data(self, stream_id: int) -> AvailableData: ... + def get_available_data(self, stream_id: int) -> AvailableDataContext: ... def get_configuration(self) -> Properties: ... def get_state(self) -> DeviceState: ... def set_configuration(self, properties: Properties) -> Properties: ... diff --git a/src/runtime.rs b/src/runtime.rs index 54f12db..a900bf7 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -182,35 +182,11 @@ impl Runtime { Python::allow_threads(py, || Ok(self.inner.execute_trigger(stream_id)?)) } - fn get_available_data(&self, stream_id: u32) -> PyResult> { - let (beg, end) = self.inner.map_read(stream_id); - let mut beg = null_mut(); - let mut end = null_mut(); - unsafe { - capi::acquire_map_read(self.as_ref().as_ptr(), stream_id, &mut beg, &mut end).ok()?; - } - let nbytes = unsafe { byte_offset_from(beg, end) }; - if nbytes > 0 { - log::trace!( - "[stream {}] ACQUIRED {:p}-{:p}:{} bytes", - stream_id, - beg, - end, - nbytes - ) - }; - Ok(if nbytes > 0 { - Some(AvailableData { - inner: Arc::new(RawAvailableData { - runtime: self.inner.clone(), - beg: NonNull::new(beg).ok_or(anyhow!("Expected non-null buffer"))?, - end: NonNull::new(end).ok_or(anyhow!("Expected non-null buffer"))?, - stream_id, - consumed_bytes: None, - }), - }) - } else { - None + fn get_available_data(&self, stream_id: u32) -> PyResult { + Ok(AvailableDataContext { + inner: self.inner.clone(), + stream_id, + available_data: None, }) } } @@ -293,27 +269,35 @@ pub(crate) struct AvailableData { #[pymethods] impl AvailableData { fn get_frame_count(&self) -> usize { - self.inner.map(|x| x.get_frame_count()).unwrap_or(0) + if let Some(inner) = &self.inner { + inner.get_frame_count() + } else { + 0 + } } - fn frames(&self) -> Option { - if let Some(frames) = self.inner { - Some(VideoFrameIterator { - store: frames.clone(), - cur: Mutex::new(frames.beg), - end: frames.end, - }) - } else { - None + fn frames(&self) -> VideoFrameIterator { + VideoFrameIterator { + inner: if let Some(frames) = &self.inner { + Some(VideoFrameIteratorInner { + store: frames.clone(), + cur: Mutex::new(frames.beg), + end: frames.end, + }) + } else { + None + }, } } fn __iter__(slf: PyRef<'_, Self>) -> PyResult> { - if let Some(frames) = self.frames() { - Py::new(slf.py(), frames) - } else { - None - } + Py::new(slf.py(), slf.frames()) + } + + fn invalidate(&mut self) { + // Will drop the RawAvailableData and cause Available data to act like + // an empty iterator. + self.inner = None; } } @@ -321,12 +305,18 @@ impl AvailableData { pub(crate) struct AvailableDataContext { inner: Arc, stream_id: u32, + available_data: Option>, } #[pymethods] impl AvailableDataContext { - fn __enter__(&self) -> PyResult> { - let AvailableDataContext { inner, stream_id } = *self; + fn __enter__(&mut self) -> PyResult>> { + let AvailableDataContext { + inner, + stream_id, + available_data, + } = self; + let stream_id = *stream_id; let (beg, end) = inner.map_read(stream_id)?; let nbytes = unsafe { byte_offset_from(beg, end) }; if nbytes > 0 { @@ -338,44 +328,43 @@ impl AvailableDataContext { nbytes ) }; - Ok(if nbytes > 0 { - Some(AvailableData { - inner: Arc::new(RawAvailableData { - runtime: self.inner.clone(), - beg: NonNull::new(beg).ok_or(anyhow!("Expected non-null buffer"))?, - end: NonNull::new(end).ok_or(anyhow!("Expected non-null buffer"))?, - stream_id, - consumed_bytes: None, - }), - }) - } else { - None - }) + if nbytes > 0 { + *available_data = Some(Python::with_gil(|py| { + Py::new( + py, + AvailableData { + inner: Some(Arc::new(RawAvailableData { + runtime: self.inner.clone(), + beg: NonNull::new(beg).ok_or(anyhow!("Expected non-null buffer"))?, + end: NonNull::new(end).ok_or(anyhow!("Expected non-null buffer"))?, + stream_id, + consumed_bytes: None, + })), + }, + ) + })?); + } + return Ok(self.available_data.clone()); } - fn __exit__(&self, _exc_type: PyAny, _exc_value: PyAny, _traceback: PyAny) {} + fn __exit__(&mut self, _exc_type: &PyAny, _exc_value: &PyAny, _traceback: &PyAny) { + Python::with_gil(|py| { + if let Some(a) = &self.available_data { + a.as_ref(py).borrow_mut().invalidate() + }; + }); + } } -#[pyclass] -struct VideoFrameIterator { +struct VideoFrameIteratorInner { store: Arc, cur: Mutex>, end: NonNull, } -unsafe impl Send for VideoFrameIterator {} +unsafe impl Send for VideoFrameIteratorInner {} -#[pymethods] -impl VideoFrameIterator { - fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { - slf - } - fn __next__(&mut self) -> Option { - self.next() - } -} - -impl Iterator for VideoFrameIterator { +impl Iterator for VideoFrameIteratorInner { type Item = VideoFrame; fn next(&mut self) -> Option { @@ -398,6 +387,29 @@ impl Iterator for VideoFrameIterator { } } +#[pyclass] +struct VideoFrameIterator { + inner: Option, +} + +#[pymethods] +impl VideoFrameIterator { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { + slf + } + fn __next__(&mut self) -> Option { + self.next() + } +} + +impl Iterator for VideoFrameIterator { + type Item = VideoFrame; + + fn next(&mut self) -> Option { + self.inner.as_mut().and_then(|it| it.next()) + } +} + #[pyclass] #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)] pub(crate) struct VideoFrameTimestamps { @@ -538,6 +550,5 @@ impl VideoFrame { } } -// TODO: Can't ensure the output array doesn't outlive the available data // TODO: Is everything really Send // TODO: mark iterable and videoframe as things that can't be shared across threads diff --git a/tests/test_basic.py b/tests/test_basic.py index d6f4c2b..871c8b3 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -104,21 +104,25 @@ def test_repeat_acq(runtime: Runtime): p = runtime.set_configuration(p) runtime.start() while True: - if a := runtime.get_available_data(0): - logging.info(f"Got {a.get_frame_count()}") - a = None - break + with runtime.get_available_data(0) as a: + if a: + logging.info(f"Got {a.get_frame_count()}") + break + assert a, "expected a not to be None" + assert a.get_frame_count()==0 + assert next(a.frames()) is None runtime.stop() - assert runtime.get_available_data(0) is None # TODO: (nclack) assert 1 acquired frame. stop should block runtime.start() while True: - if a := runtime.get_available_data(0): - logging.info(f"Got {a.get_frame_count()}") - a = None - break + with runtime.get_available_data(0) as a: + if a: + logging.info(f"Got {a.get_frame_count()}") + break + assert a, "expected a not to be None" + assert a.get_frame_count()==0 + assert next(a.frames()) is None runtime.stop() - assert runtime.get_available_data(0) is None # TODO: (nclack) assert 1 more acquired frame. stop cancels and waits. From 3ce9e05ad6b24f3ee1c2e5dead3ed7d723d2df8f Mon Sep 17 00:00:00 2001 From: Nathan Clack Date: Tue, 28 Nov 2023 19:32:42 -0800 Subject: [PATCH 03/12] (wip) --- tests/test_basic.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_basic.py b/tests/test_basic.py index 871c8b3..25b9ea4 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -96,8 +96,8 @@ def test_zero_conf_start(runtime: Runtime): def test_repeat_acq(runtime: Runtime): p = acquire.setup(runtime, "simulated: radial sin", "Trash") - assert p.video[0].camera.identifier is not None - assert p.video[0].storage.identifier is not None + assert p.video[0].camera.identifier is not None, "Expected a camera identifier" + assert p.video[0].storage.identifier is not None, "Expected a storage identifier" assert p.video[0].storage.settings.filename == "out.tif" p.video[0].camera.settings.shape = (192, 108) p.video[0].max_frame_count = 10 @@ -108,9 +108,9 @@ def test_repeat_acq(runtime: Runtime): if a: logging.info(f"Got {a.get_frame_count()}") break - assert a, "expected a not to be None" - assert a.get_frame_count()==0 - assert next(a.frames()) is None + if a: + assert a.get_frame_count()==0 + assert next(a.frames()) is None runtime.stop() # TODO: (nclack) assert 1 acquired frame. stop should block runtime.start() @@ -119,9 +119,9 @@ def test_repeat_acq(runtime: Runtime): if a: logging.info(f"Got {a.get_frame_count()}") break - assert a, "expected a not to be None" - assert a.get_frame_count()==0 - assert next(a.frames()) is None + if a: + assert a.get_frame_count()==0 + assert next(a.frames()) is None runtime.stop() # TODO: (nclack) assert 1 more acquired frame. stop cancels and waits. From 74f6609bcc1f411275b7da2700b2c317b28a8acf Mon Sep 17 00:00:00 2001 From: Nathan Clack Date: Tue, 28 Nov 2023 20:32:57 -0800 Subject: [PATCH 04/12] close but not quite right --- python/acquire/__init__.py | 23 ++++---- tests/test_basic.py | 111 +++++++++++++++++++------------------ 2 files changed, 69 insertions(+), 65 deletions(-) diff --git a/python/acquire/__init__.py b/python/acquire/__init__.py index e76f3e8..adf6627 100644 --- a/python/acquire/__init__.py +++ b/python/acquire/__init__.py @@ -189,17 +189,18 @@ def is_not_done() -> bool: def next_frame() -> Optional[npt.NDArray[Any]]: """Get the next frame from the current stream.""" if nframes[stream_id] < p.video[stream_id].max_frame_count: - if packet := runtime.get_available_data(stream_id): - n = packet.get_frame_count() - nframes[stream_id] += n - logging.info( - f"[stream {stream_id}] frame count: {nframes}" - ) - f = next(packet.frames()) - logging.debug( - f"stream {stream_id} frame {f.metadata().frame_id}" - ) - return f.data().squeeze().copy() + with runtime.get_available_data(stream_id) as packet: + if packet: + n = packet.get_frame_count() + nframes[stream_id] += n + logging.info( + f"[stream {stream_id}] frame count: {nframes}" + ) + f = next(packet.frames()) + logging.debug( + f"stream {stream_id} frame {f.metadata().frame_id}" + ) + return f.data().squeeze().copy() return None while is_not_done(): # runtime.get_state()==DeviceState.Running: diff --git a/tests/test_basic.py b/tests/test_basic.py index 25b9ea4..d2f2c29 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -3,7 +3,7 @@ import time from datetime import timedelta from time import sleep -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple import acquire import dask.array as da @@ -11,7 +11,7 @@ import pytest import tifffile import zarr -from acquire.acquire import DeviceKind, DeviceState, Runtime, Trigger +from acquire.acquire import DeviceKind, DeviceState, Runtime, Trigger, VideoFrame from ome_zarr.io import parse_url from ome_zarr.reader import Reader from skimage.transform import downscale_local_mean @@ -139,10 +139,10 @@ def test_repeat_with_no_stop(runtime: Runtime): runtime.start() # wait for 1 frame while True: - if a := runtime.get_available_data(0): - logging.info(f"Got {a.get_frame_count()}") - a = None - break + with runtime.get_available_data(0) as a: + if a: + logging.info(f"Got {a.get_frame_count()} frame") + break # acq is still on going here with pytest.raises(RuntimeError): logging.info("Next start should fail gracefully") @@ -189,18 +189,19 @@ def took_too_long(): while nframes < p.video[0].max_frame_count and not took_too_long(): clock = time.time() - if a := runtime.get_available_data(0): - packet = a.get_frame_count() - for f in a.frames(): + with runtime.get_available_data(0) as a: + if a: + packet = a.get_frame_count() + for f in a.frames(): + logging.info( + f"{f.data().shape} {f.data()[0][0][0][0]} {f.metadata()}" + ) + # del f # <-- fails to get the last frames if this is held? + # del a # <-- fails to get the last frames if this is held? + nframes += packet logging.info( - f"{f.data().shape} {f.data()[0][0][0][0]} {f.metadata()}" + f"frame count: {nframes} - frames in packet: {packet}" ) - del f # <-- fails to get the last frames if this is held? - del a # <-- fails to get the last frames if this is held? - nframes += packet - logging.info( - f"frame count: {nframes} - frames in packet: {packet}" - ) elapsed = time.time() - clock sleep(max(0, 0.1 - elapsed)) @@ -239,9 +240,10 @@ def test_change_filename(runtime: Runtime): nframes = 0 runtime.start() while nframes < p.video[0].max_frame_count: - if packet := runtime.get_available_data(0): - nframes += packet.get_frame_count() - packet = None + with runtime.get_available_data(0) as packet: + if packet: + nframes += packet.get_frame_count() + packet = None logging.info("Stopping") runtime.stop() @@ -265,9 +267,9 @@ def test_write_external_metadata_to_tiff( nframes = 0 runtime.start() while nframes < p.video[0].max_frame_count: - if packet := runtime.get_available_data(0): - nframes += packet.get_frame_count() - packet = None + with runtime.get_available_data(0) as packet: + if packet: + nframes += packet.get_frame_count() runtime.stop() # Check that the written tif has the expected structure @@ -310,9 +312,9 @@ def test_write_external_metadata_to_zarr( nframes = 0 runtime.start() while nframes < p.video[0].max_frame_count: - if packet := runtime.get_available_data(0): - nframes += packet.get_frame_count() - packet = None + with runtime.get_available_data(0) as packet: + if packet: + nframes += packet.get_frame_count() runtime.stop() assert p.video[0].storage.settings.filename @@ -631,19 +633,20 @@ def is_not_done() -> bool: stream_id = 0 while is_not_done(): if nframes[stream_id] < p.video[stream_id].max_frame_count: - if packet := runtime.get_available_data(stream_id): - n = packet.get_frame_count() - for i, frame in enumerate(packet.frames()): - expected_frame_id = nframes[stream_id] + i - assert frame.metadata().frame_id == expected_frame_id, ( - "frame id's didn't match " - + f"({frame.metadata().frame_id}!={expected_frame_id})" - + f" [stream {stream_id} nframes {nframes}]" - ) - del frame - del packet - nframes[stream_id] += n - logging.debug(f"NFRAMES {nframes}") + with runtime.get_available_data(stream_id) as packet: + if packet: + n = packet.get_frame_count() + for i, frame in enumerate(packet.frames()): + expected_frame_id = nframes[stream_id] + i + assert frame.metadata().frame_id == expected_frame_id, ( + "frame id's didn't match " + + f"({frame.metadata().frame_id}!={expected_frame_id})" + + f" [stream {stream_id} nframes {nframes}]" + ) + del frame + del packet + nframes[stream_id] += n + logging.debug(f"NFRAMES {nframes}") stream_id = (stream_id + 1) % 2 logging.info("Stopping") @@ -669,10 +672,12 @@ def test_abort(runtime: Runtime): logging.info("Aborting") runtime.abort() - while packet := runtime.get_available_data(0): - nframes += packet.get_frame_count() - - del packet + while True: + with runtime.get_available_data(0) as packet: + if packet: + nframes += packet.get_frame_count() + else: + break logging.debug( f"Frames expected: {p.video[0].max_frame_count}, actual: {nframes}" @@ -680,15 +685,17 @@ def test_abort(runtime: Runtime): assert nframes < p.video[0].max_frame_count -def wait_for_data(runtime: Runtime, stream_id: int = 0, timeout: Optional[timedelta] = None)-> acquire.AvailableData: +def wait_for_data(runtime: Runtime, stream_id: int = 0, timeout: Optional[timedelta] = None)-> Tuple[int,int]: # None is used as a missing sentinel value, not to indicate no timeout. if timeout is None: timeout = timedelta(seconds=5) sleep_duration = timedelta(microseconds=10000) elapsed = timedelta() while elapsed < timeout: - if packet := runtime.get_available_data(stream_id): - return packet + with runtime.get_available_data(stream_id) as packet: + if packet: + frames = list(packet.frames()) + return (len(frames), frames[0].metadata().frame_id) sleep(sleep_duration.total_seconds()) elapsed += sleep_duration raise RuntimeError(f"Timed out waiting for condition after {elapsed.total_seconds()} seconds.") @@ -714,23 +721,19 @@ def test_execute_trigger(runtime: Runtime): runtime.start() # No triggers yet, so no data. - assert runtime.get_available_data(0) is None + with runtime.get_available_data(0) as packet: + assert packet is None # Snap a few individual frames for i in range(p.video[0].max_frame_count): runtime.execute_trigger(0) - packet = wait_for_data(runtime, 0) - frames = tuple(packet.frames()) - assert packet.get_frame_count() == 1 - assert frames[0].metadata().frame_id == i - del frames - del packet + count, frame_id = wait_for_data(runtime, 0) + assert count == 1 + assert frame_id == i runtime.stop() -# FIXME: (nclack) awkwardness around references (available frames, f) - # NOTES: # # With pytest, use `--log-cli-level=0` to see the lowest level logs. From 14bebdc9fa00ad5c06af753141d211e6e0b066cb Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 12 Dec 2023 09:02:31 -0500 Subject: [PATCH 05/12] Update test_write_external_metadata_to_zarr to use new AvailableData context. --- tests/test_zarr.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_zarr.py b/tests/test_zarr.py index 6511805..18bc4b4 100644 --- a/tests/test_zarr.py +++ b/tests/test_zarr.py @@ -46,9 +46,10 @@ def test_write_external_metadata_to_zarr( nframes = 0 runtime.start() while nframes < p.video[0].max_frame_count: - if packet := runtime.get_available_data(0): - nframes += packet.get_frame_count() - packet = None + with runtime.get_available_data(0) as packet: + if packet: + nframes += packet.get_frame_count() + packet = None runtime.stop() assert p.video[0].storage.settings.filename From 69f74d1286584a11deaa55ade5dde38848c6cf97 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 12 Dec 2023 09:35:06 -0500 Subject: [PATCH 06/12] Updates for stubtest. --- python/acquire/acquire.pyi | 7 +++++-- src/lib.rs | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/acquire/acquire.pyi b/python/acquire/acquire.pyi index cea5517..882c717 100644 --- a/python/acquire/acquire.pyi +++ b/python/acquire/acquire.pyi @@ -14,13 +14,16 @@ from numpy.typing import NDArray @final class AvailableDataContext: - def __enter__(self)->Optional[AvailableData]: ... - def __exit__(self,exc_type:Any,exc_value:Any,traceback:Any)->None: ... + def __enter__(self) -> Optional[AvailableData]: ... + def __exit__( + self, exc_type: Any, exc_value: Any, traceback: Any + ) -> None: ... @final class AvailableData: def frames(self) -> Iterator[VideoFrame]: ... def get_frame_count(self) -> int: ... + def invalidate(self) -> None: ... def __iter__(self) -> Iterator[VideoFrame]: ... @final diff --git a/src/lib.rs b/src/lib.rs index fe49c94..276ce3d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,7 @@ use device_manager::DeviceManager; use pyo3::prelude::*; use std::ffi::CStr; -use crate::runtime::{AvailableData, VideoFrame, VideoFrameMetadata, VideoFrameTimestamps}; +use crate::runtime::{AvailableData, AvailableDataContext, VideoFrame, VideoFrameMetadata, VideoFrameTimestamps}; trait Status: Copy + Sized { fn is_ok(&self) -> bool; @@ -60,6 +60,7 @@ fn acquire(py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; From 2661fb13970411cc9078a913d623f65f702ced1e Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Tue, 12 Dec 2023 09:42:42 -0500 Subject: [PATCH 07/12] Update minor release version. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 4e2adf6..44c228a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "acquire-imaging" authors = ["Nathan Clack "] -version = "0.2.1" +version = "0.3.0" edition = "2021" [lib] From d780af68391a5d9e9d0655b24b8f3805cfbd4055 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Wed, 13 Dec 2023 10:26:34 -0500 Subject: [PATCH 08/12] Remove explicit `del packet` statements as no longer necessary. --- tests/test_basic.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_basic.py b/tests/test_basic.py index 7ffd793..ea67368 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -183,8 +183,6 @@ def took_too_long(): logging.info( f"{f.data().shape} {f.data()[0][0][0][0]} {f.metadata()}" ) - # del f # <-- fails to get the last frames if this is held? - # del a # <-- fails to get the last frames if this is held? nframes += packet logging.info( f"frame count: {nframes} - frames in packet: {packet}" @@ -230,7 +228,6 @@ def test_change_filename(runtime: Runtime): with runtime.get_available_data(0) as packet: if packet: nframes += packet.get_frame_count() - packet = None logging.info("Stopping") runtime.stop() @@ -328,8 +325,6 @@ def is_not_done() -> bool: + f"({frame.metadata().frame_id}!={expected_frame_id})" + f" [stream {stream_id} nframes {nframes}]" ) - del frame - del packet nframes[stream_id] += n logging.debug(f"NFRAMES {nframes}") From e4a96173251e4f321500d2c1f196a332e16e3bfc Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Wed, 13 Dec 2023 10:28:06 -0500 Subject: [PATCH 09/12] Remove one more `packet = None` statement. --- tests/test_zarr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_zarr.py b/tests/test_zarr.py index 18bc4b4..a17b9b6 100644 --- a/tests/test_zarr.py +++ b/tests/test_zarr.py @@ -49,7 +49,6 @@ def test_write_external_metadata_to_zarr( with runtime.get_available_data(0) as packet: if packet: nframes += packet.get_frame_count() - packet = None runtime.stop() assert p.video[0].storage.settings.filename From abb094d74004fcfd3b1b51f8591b08ad36129656 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Wed, 13 Dec 2023 11:12:32 -0500 Subject: [PATCH 10/12] Run formatters. --- src/lib.rs | 4 +++- tests/test_basic.py | 16 +++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 1510e79..ec6f344 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,9 @@ use device_manager::DeviceManager; use pyo3::prelude::*; use std::ffi::CStr; -use crate::runtime::{AvailableData, AvailableDataContext, VideoFrame, VideoFrameMetadata, VideoFrameTimestamps}; +use crate::runtime::{ + AvailableData, AvailableDataContext, VideoFrame, VideoFrameMetadata, VideoFrameTimestamps, +}; trait Status: Copy + Sized { fn is_ok(&self) -> bool; diff --git a/tests/test_basic.py b/tests/test_basic.py index ea67368..b19ccad 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -83,8 +83,12 @@ def test_zero_conf_start(runtime: Runtime): def test_repeat_acq(runtime: Runtime): p = acquire.setup(runtime, "simulated: radial sin", "Trash") - assert p.video[0].camera.identifier is not None, "Expected a camera identifier" - assert p.video[0].storage.identifier is not None, "Expected a storage identifier" + assert ( + p.video[0].camera.identifier is not None + ), "Expected a camera identifier" + assert ( + p.video[0].storage.identifier is not None + ), "Expected a storage identifier" assert p.video[0].storage.settings.filename == "out.tif" p.video[0].camera.settings.shape = (192, 108) p.video[0].max_frame_count = 10 @@ -96,7 +100,7 @@ def test_repeat_acq(runtime: Runtime): logging.info(f"Got {a.get_frame_count()}") break if a: - assert a.get_frame_count()==0 + assert a.get_frame_count() == 0 assert next(a.frames()) is None runtime.stop() # TODO: (nclack) assert 1 acquired frame. stop should block @@ -107,7 +111,7 @@ def test_repeat_acq(runtime: Runtime): logging.info(f"Got {a.get_frame_count()}") break if a: - assert a.get_frame_count()==0 + assert a.get_frame_count() == 0 assert next(a.frames()) is None runtime.stop() # TODO: (nclack) assert 1 more acquired frame. stop cancels and waits. @@ -320,7 +324,9 @@ def is_not_done() -> bool: n = packet.get_frame_count() for i, frame in enumerate(packet.frames()): expected_frame_id = nframes[stream_id] + i - assert frame.metadata().frame_id == expected_frame_id, ( + assert ( + frame.metadata().frame_id == expected_frame_id + ), ( "frame id's didn't match " + f"({frame.metadata().frame_id}!={expected_frame_id})" + f" [stream {stream_id} nframes {nframes}]" From f4a573dea1ea0dccb46f535769635146927464a5 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Wed, 13 Dec 2023 11:12:58 -0500 Subject: [PATCH 11/12] Update acquire-video-runtime. --- acquire-libs/acquire-video-runtime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acquire-libs/acquire-video-runtime b/acquire-libs/acquire-video-runtime index 83bd825..8ff1105 160000 --- a/acquire-libs/acquire-video-runtime +++ b/acquire-libs/acquire-video-runtime @@ -1 +1 @@ -Subproject commit 83bd825626e6530895598ee1f78c9cb2230d9b40 +Subproject commit 8ff1105935ba1f952bdfd436543e0e1b3ea885f6 From ff23b80244e3546c3857cc7b8bb83e9091183fa1 Mon Sep 17 00:00:00 2001 From: Alan Liddell Date: Wed, 13 Dec 2023 14:36:12 -0500 Subject: [PATCH 12/12] make flake8 happy --- tests/test_basic.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_basic.py b/tests/test_basic.py index b19ccad..09701d8 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -185,7 +185,8 @@ def took_too_long(): packet = a.get_frame_count() for f in a.frames(): logging.info( - f"{f.data().shape} {f.data()[0][0][0][0]} {f.metadata()}" + f"{f.data().shape} {f.data()[0][0][0][0]} " + + f"{f.metadata()}" ) nframes += packet logging.info( @@ -328,7 +329,8 @@ def is_not_done() -> bool: frame.metadata().frame_id == expected_frame_id ), ( "frame id's didn't match " - + f"({frame.metadata().frame_id}!={expected_frame_id})" + + f"({frame.metadata().frame_id}" + + f"!={expected_frame_id})" + f" [stream {stream_id} nframes {nframes}]" ) nframes[stream_id] += n