From 54f4d2ad6c329a1809904dae3030ae757a2b3db0 Mon Sep 17 00:00:00 2001 From: Erol444 Date: Wed, 9 Oct 2024 21:34:11 +0200 Subject: [PATCH] Fixed depth recording in video format (ffv1 codec). Added example to showcase that --- apps/record/requirements.txt | 2 +- apps/uvc/requirements.txt | 3 +- .../examples/recording/depth_video_record.py | 30 +++++++++++++++++++ .../recorders/video_writers/video_writer.py | 8 +++-- 4 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 depthai_sdk/examples/recording/depth_video_record.py diff --git a/apps/record/requirements.txt b/apps/record/requirements.txt index 8e00ed6d1..786d5213f 100644 --- a/apps/record/requirements.txt +++ b/apps/record/requirements.txt @@ -1,6 +1,6 @@ depthai==2.17.0.0 depthai-sdk==1.2.4 # New SDK has Record module completely refactored. TODO port this app to new SDK -numpy==1.23.4 +numpy==1.26.4 av==9.2.0 mcap==0.0.10 mcap-ros1-support==0.0.8 diff --git a/apps/uvc/requirements.txt b/apps/uvc/requirements.txt index b27a39b71..d4ddc2a52 100644 --- a/apps/uvc/requirements.txt +++ b/apps/uvc/requirements.txt @@ -1,3 +1,4 @@ --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-snapshot-local/ depthai==2.19.1.0.dev+e88af9a9db3dc630a3011dd52d1f5700cf6bf9b8 -depthai-sdk==1.2.1 \ No newline at end of file +depthai-sdk==1.2.1 +numpy==1.26.4 \ No newline at end of file diff --git a/depthai_sdk/examples/recording/depth_video_record.py b/depthai_sdk/examples/recording/depth_video_record.py new file mode 100644 index 000000000..3f238e30e --- /dev/null +++ b/depthai_sdk/examples/recording/depth_video_record.py @@ -0,0 +1,30 @@ +from depthai_sdk import OakCamera, RecordType +import depthai as dai + +FPS = 30 +with OakCamera() as oak: + color = oak.create_camera('CAM_A', resolution='1080p', encode='mjpeg', fps=FPS) + color.config_color_camera(isp_scale=(2,3)) # 720P color frames, mjpeg + + stereo = oak.create_stereo(resolution='720p', fps=FPS) + stereo.config_stereo(align=color, subpixel=True, lr_check=True) + stereo.node.setOutputSize(640, 360) # 720p, downscaled to 640x360 (decimation filter, median filtering) + # On-device post processing for stereo depth + config = stereo.node.initialConfig.get() + stereo.node.setPostProcessingHardwareResources(3, 3) + config.postProcessing.speckleFilter.enable = True + config.postProcessing.thresholdFilter.minRange = 400 + config.postProcessing.thresholdFilter.maxRange = 10_000 # 10m + config.postProcessing.decimationFilter.decimationFactor = 2 + config.postProcessing.decimationFilter.decimationMode = dai.StereoDepthConfig.PostProcessing.DecimationFilter.DecimationMode.NON_ZERO_MEDIAN + stereo.node.initialConfig.set(config) + """ + Depth will get encoded on-host with FFV1 codec (it supports 16bit grey), and color will be + encoded on-device with MJPEG codec. FFV1 works well with .avi container, so depth video will be + saved as .avi, and color video will be saved as .mp4. + """ + record_components = [stereo.out.depth, color.out.encoded] + oak.record(record_components, './', record_type=RecordType.VIDEO).configure_syncing(True, threshold_ms=500/30) + oak.start(blocking=True) + + diff --git a/depthai_sdk/src/depthai_sdk/recorders/video_writers/video_writer.py b/depthai_sdk/src/depthai_sdk/recorders/video_writers/video_writer.py index b0b0f731d..bff5e0454 100644 --- a/depthai_sdk/src/depthai_sdk/recorders/video_writers/video_writer.py +++ b/depthai_sdk/src/depthai_sdk/recorders/video_writers/video_writer.py @@ -60,7 +60,7 @@ def _create_file(self, path_to_file: str, frame: dai.ImgFrame): options = {} if self._lossless: self._fourcc = 'rawvideo' - elif frame.getType() == dai.ImgFrame.Type.RAW16: + elif frame.getType() == dai.ImgFrame.Type.RAW16: # Depth map self._fourcc = 'ffv1' self._format = 'gray16le' else: # Mono/Color, encode @@ -74,6 +74,11 @@ def _create_file(self, path_to_file: str, frame: dai.ImgFrame): self._stream.codec_context.width = frame.getWidth() self._stream.codec_context.height = frame.getHeight() + if self._fourcc == 'ffv1': + self._stream.width = frame.getWidth() + self._stream.height = frame.getHeight() + self._stream.pix_fmt = 'gray16le' # Required for depth recording to work correctly + def write(self, img_frame: dai.ImgFrame): if self._file is None: self.create_file(subfolder='', frame=img_frame) @@ -90,7 +95,6 @@ def write(self, img_frame: dai.ImgFrame): video_format = 'gray16le' else: raise ValueError(f'Unsupported frame type: {img_frame.getType()}') - video_frame = av.VideoFrame.from_ndarray(img_frame.getFrame(), format=video_format) ts = int((img_frame.getTimestampDevice() - self._start_ts).total_seconds() * 1e3) # To milliseconds