[BUG] .setTimestamp() inside Script node only sets the deviceTimestamp

[jakaskerl]

Setting timestamp via ImgFrame.setTimestamp() only sets device timestamp. When read on host side, via getTimestamp() the value is unaffected. getTimestampDevice() works as expected.

Repro code:

import depthai as dai
import cv2
from math import isclose

# Setup the pipeline
pipeline = dai.Pipeline()

# Create nodes for RGB and mono streams
rgbCamera = pipeline.create(dai.node.ColorCamera)
monoCamera = pipeline.create(dai.node.MonoCamera)

# Set camera properties
rgbCamera.setBoardSocket(dai.CameraBoardSocket.RGB)
monoCamera.setBoardSocket(dai.CameraBoardSocket.LEFT)
rgbCamera.setResolution(dai.ColorCameraProperties.SensorResolution.THE_1080_P)
monoCamera.setResolution(dai.MonoCameraProperties.SensorResolution.THE_720_P)


# Create a script node
script = pipeline.create(dai.node.Script)
script.setScript("""
import time
stream1_in = node.io['rgb']
stream2_in = node.io['mono']
stream1_out = node.io['synced_rgb']
stream2_out = node.io['synced_mono']

while True:
    frame1 = stream1_in.get()
    frame2 = stream2_in.get()

    timestamp1 = frame1.getTimestamp()
    timestamp2 = frame2.getTimestamp()
    node.warn(f"RGB Frame Timestamp: {timestamp1}")
    node.warn(f"Mono Frame Timestamp: {timestamp2}")

    # Set the RGB frame's timestamp to match the mono frame's timestamp
    frame1.setTimestamp(timestamp2)
    frame1.setSequenceNum(frame2.getSequenceNum())
                 
    node.warn(f"RGB Frame Timestamp after sync: {frame1.getTimestamp()}")
    node.warn(f"Mono Frame Timestamp after sync: {frame2.getTimestamp()}")
                 
    # Send the synchronized frames to the output
    stream1_out.send(frame1)
    stream2_out.send(frame2)
                 
    
""")

# Link the outputs to the script node
rgbCamera.video.link(script.inputs['rgb'])
monoCamera.out.link(script.inputs['mono'])

# Create XLinkOut nodes to output the synchronized streams
xoutSyncedRgb = pipeline.create(dai.node.XLinkOut)
xoutSyncedRgb.setStreamName("synced_rgb")
script.outputs['synced_rgb'].link(xoutSyncedRgb.input)

xoutSyncedmono = pipeline.create(dai.node.XLinkOut)
xoutSyncedmono.setStreamName("synced_mono")
script.outputs['synced_mono'].link(xoutSyncedmono.input)

# Connect to device and start pipeline
with dai.Device(pipeline) as device:
    rgbQueue = device.getOutputQueue(name="synced_rgb", maxSize=8, blocking=False)
    monoQueue = device.getOutputQueue(name="synced_mono", maxSize=8, blocking=False)

    while True:
        syncedRgbFrame = rgbQueue.get()
        syncedMonoFrame = monoQueue.get()
    
        # Get timestamps after synchronization
        rgbTimestamp = syncedRgbFrame.getTimestamp().total_seconds()
        monoTimestamp = syncedMonoFrame.getTimestamp().total_seconds()

        # Print the synchronized timestamps
        print(f"Synced RGB Frame Timestamp: {rgbTimestamp}")
        print(f"Synced mono Frame Timestamp: {monoTimestamp}")

        print(f"RGB Frame Device timestamp: {syncedRgbFrame.getTimestampDevice().total_seconds()}")
        print(f"Mono Frame Device timestamp: {syncedMonoFrame.getTimestampDevice().total_seconds()}")

        # Check if the timestamps differ too much
        if not isclose(rgbTimestamp, monoTimestamp, abs_tol=0.000001):
            raise SystemExit(f"Warning: Timestamps differ more than expected: RGB {rgbTimestamp}, mono {monoTimestamp}")

        # Break the loop if 'q' is pressed
        if cv2.waitKey(1) == ord('q'):
            break