Improve remote capture.

configs/demo.yaml

@@ -30,23 +30,26 @@ display:
   white: False
 
 capture:
-  sensor: rpi_hq
   gamma: null # for visualization
-  exp: 0.02
-  delay: 2
   script: ~/LenslessPiCam/scripts/measure/on_device_capture.py
-  iso: 100
+  delay: 2
+
+  # -- capture settings
+  sensor: rpi_hq
+  legacy: True   # Legacy image capture software (raspistill) or new one (libcamera)
+  bayer: True    # whether to capture Bayer data, otherwise directly capture RGB (demosaiced) data
+  nbits_capture: 12   # bit-depth to capture, check support bit depth of sensor: lensless/hardware/sensor.py
+  raw_data_fn: raw_data    # output file name
+  exp: 0.02
   config_pause: 2
-  sensor_mode: "0"
-  # nbits_out: 12
-  nbits_out: 8    # light data transer, doesn't seem to worsen performance
-  nbits: 12
-  legacy: True
+  sensor_mode: "0"   # {'off': 0, 'auto': 1, 'sunlight': 2, 'cloudy': 3, 'shade': 4, 'tungsten': 5, 'fluorescent': 6, 'incandescent': 7, 'flash': 8, 'horizon': 9}
+  iso: 100
+
+  # -- if bayer=True, capture Bayer data but return as RGB or grayscale
   gray: False     # only for legacy=True, if bayer=True, remote script returns grayscale data
   # rgb: False   # only for legacy=True, if bayer=True, remote script return RGB data
-  raw_data_fn: raw_data
-  bayer: True
-  source: white
+  # nbits_out: 12
+  nbits_out: 8   # used if gray=True or rgb=True
 
   # remote script return Bayer data, full res
   # awb_gains: null

lensless/hardware/utils.py

@@ -21,12 +21,13 @@ def check_capture_config(config):
 
     sensor = config.sensor
     nbits_capture = config.nbits_capture
+    nbits_out = config.nbits_out
     exp = config.exp
     rgb = config.rgb
     gray = config.gray
     legacy = config.legacy
     down = config.down
-    res = config.res
+    res = config.res if "res" in config.keys() else None
     awb_gains = config.awb_gains
 
     assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}"
@@ -42,6 +43,8 @@ def check_capture_config(config):
         nbits_capture in supported_bit_depth
     ), f"nbits_capture must be one of {supported_bit_depth} for sensor {sensor}"
 
+    assert nbits_capture >= nbits_out, "nbits_capture must be greater than or equal to nbits_out"
+
     if SensorParam.BLACK_LEVEL in sensor_dict[sensor]:
         black_level = sensor_dict[sensor][SensorParam.BLACK_LEVEL] * (2**nbits_capture - 1)
     else:

lensless/utils/io.py

@@ -39,6 +39,10 @@ def load_image(
     """
     Load image as numpy array.
 
+    Note that for bayer data input, the image is converted to RGB using
+    color correction matrix (CCM) and black level subtraction parameters
+    for the Raspberry Pi HQ camera (12-bit).
+
     Parameters
     ----------
     fp : str
@@ -122,12 +126,16 @@ def load_image(
 
     if bayer:
         assert len(img.shape) == 2, img.shape
+
+        # assume RPi HQ camera
         if nbits is None:
             if img.max() > 255:
-                # HQ camera
                 nbits = 12
             else:
                 nbits = 8
+                # convert black level from 12 bit to 8 bit
+                if black_level > 255:
+                    black_level = black_level / (2**12 - 1) * 255
 
         if back:
             back_img = cv2.imread(back, cv2.IMREAD_UNCHANGED)

scripts/measure/on_device_capture.py

@@ -33,7 +33,6 @@
 from PIL import Image
 from lensless.hardware.utils import get_distro, check_capture_config
 from lensless.utils.image import bayer2rgb_cc, rgb2gray, resize
-from lensless.hardware.sensor import SensorOptions, sensor_dict, SensorParam
 from fractions import Fraction
 import time
 

scripts/measure/remote_capture.py

@@ -33,8 +33,7 @@
 from pprint import pprint
 import matplotlib.pyplot as plt
 import rawpy
-from lensless.hardware.utils import check_username_hostname
-from lensless.hardware.sensor import SensorOptions, sensor_dict, SensorParam
+from lensless.hardware.utils import check_username_hostname, check_capture_config
 from lensless.utils.image import rgb2gray, print_image_info
 from lensless.utils.plot import plot_image, pixel_histogram
 from lensless.utils.io import save_image
@@ -44,30 +43,21 @@
 @hydra.main(version_base=None, config_path="../../configs", config_name="demo")
 def liveview(config):
 
-    sensor = config.capture.sensor
-    assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}"
+    username = config.rpi.username
+    hostname = config.rpi.hostname
+    check_username_hostname(username, hostname)
 
+    black_level, ccm, _ = check_capture_config(config.capture)
+    sensor = config.capture.sensor
     bayer = config.capture.bayer
     rgb = config.capture.rgb
     gray = config.capture.gray
-
-    check_username_hostname(config.rpi.username, config.rpi.hostname)
-    username = config.rpi.username
-    hostname = config.rpi.hostname
     legacy = config.capture.legacy
     nbits_out = config.capture.nbits_out
     fn = config.capture.raw_data_fn
     gamma = config.capture.gamma
-    source = config.capture.source
     plot = config.plot
 
-    assert (
-        nbits_out in sensor_dict[sensor][SensorParam.BIT_DEPTH]
-    ), f"capture.nbits_out must be one of {sensor_dict[sensor][SensorParam.BIT_DEPTH]} for sensor {sensor}"
-    assert (
-        config.capture.nbits in sensor_dict[sensor][SensorParam.BIT_DEPTH]
-    ), f"capture.nbits must be one of {sensor_dict[sensor][SensorParam.BIT_DEPTH]} for sensor {sensor}"
-
     if config.save:
         if config.output is not None:
             # make sure output directory exists
@@ -84,10 +74,8 @@ def liveview(config):
     pic_command = (
         f"{config.rpi.python} {config.capture.script} sensor={sensor} bayer={bayer} fn={remote_fn} exp={config.capture.exp} iso={config.capture.iso} "
         f"config_pause={config.capture.config_pause} sensor_mode={config.capture.sensor_mode} nbits_out={config.capture.nbits_out} "
-        f"legacy={config.capture.legacy} rgb={config.capture.rgb} gray={config.capture.gray} "
+        f"legacy={config.capture.legacy} rgb={config.capture.rgb} gray={config.capture.gray} nbits_capture={config.capture.nbits_capture} "
     )
-    if config.capture.nbits > 8:
-        pic_command += " sixteen=True"
     if config.capture.down:
         pic_command += f" down={config.capture.down}"
     if config.capture.awb_gains:
@@ -232,6 +220,8 @@ def liveview(config):
                 blue_gain=blue_gain,
                 red_gain=red_gain,
                 nbits_out=nbits_out,
+                black_level=black_level,
+                ccm=ccm,
             )
 
             # write RGB data
@@ -250,30 +240,20 @@ def liveview(config):
             if save:
                 plt.savefig(os.path.join(save, f"{fn}_plot.png"))
 
-            # plot red channel
-            if source == "red":
-                img_1chan = img[:, :, 0]
-            elif source == "green":
-                img_1chan = img[:, :, 1]
-            elif source == "blue":
-                img_1chan = img[:, :, 2]
-            else:
-                img_1chan = rgb2gray(img[None, :, :, :])
+            # plot grayscale
+            img_1chan = rgb2gray(img[None, :, :, :])
             ax = plot_image(img_1chan)
-            if source == "white":
-                ax.set_title("Gray scale")
-            else:
-                ax.set_title(f"{source} channel")
+            ax.set_title("Grayscale")
             if save:
-                plt.savefig(os.path.join(save, f"{fn}_1chan.png"))
+                plt.savefig(os.path.join(save, f"{fn}_gray.png"))
 
             # plot histogram, useful for checking clipping
             pixel_histogram(img)
             if save:
                 plt.savefig(os.path.join(save, f"{fn}_hist.png"))
             pixel_histogram(img_1chan)
             if save:
-                plt.savefig(os.path.join(save, f"{fn}_1chan_hist.png"))
+                plt.savefig(os.path.join(save, f"{fn}_gray_hist.png"))
 
         else:
             ax = plot_image(img, gamma=gamma)