From c9e7ce448ea679e71057721954513fdebc9f06a5 Mon Sep 17 00:00:00 2001 From: Eric Bezzam Date: Wed, 29 Nov 2023 17:52:00 +0100 Subject: [PATCH] Improve remote capture. --- configs/demo.yaml | 27 +++++++++------- lensless/hardware/utils.py | 5 ++- lensless/utils/io.py | 10 +++++- scripts/measure/on_device_capture.py | 1 - scripts/measure/remote_capture.py | 48 ++++++++-------------------- 5 files changed, 42 insertions(+), 49 deletions(-) diff --git a/configs/demo.yaml b/configs/demo.yaml index 47f13ba4..698e73be 100644 --- a/configs/demo.yaml +++ b/configs/demo.yaml @@ -30,23 +30,26 @@ display: white: False capture: - sensor: rpi_hq gamma: null # for visualization - exp: 0.02 - delay: 2 script: ~/LenslessPiCam/scripts/measure/on_device_capture.py - iso: 100 + delay: 2 + + # -- capture settings + sensor: rpi_hq + legacy: True # Legacy image capture software (raspistill) or new one (libcamera) + bayer: True # whether to capture Bayer data, otherwise directly capture RGB (demosaiced) data + nbits_capture: 12 # bit-depth to capture, check support bit depth of sensor: lensless/hardware/sensor.py + raw_data_fn: raw_data # output file name + exp: 0.02 config_pause: 2 - sensor_mode: "0" - # nbits_out: 12 - nbits_out: 8 # light data transer, doesn't seem to worsen performance - nbits: 12 - legacy: True + sensor_mode: "0" # {'off': 0, 'auto': 1, 'sunlight': 2, 'cloudy': 3, 'shade': 4, 'tungsten': 5, 'fluorescent': 6, 'incandescent': 7, 'flash': 8, 'horizon': 9} + iso: 100 + + # -- if bayer=True, capture Bayer data but return as RGB or grayscale gray: False # only for legacy=True, if bayer=True, remote script returns grayscale data # rgb: False # only for legacy=True, if bayer=True, remote script return RGB data - raw_data_fn: raw_data - bayer: True - source: white + # nbits_out: 12 + nbits_out: 8 # used if gray=True or rgb=True # remote script return Bayer data, full res # awb_gains: null diff --git a/lensless/hardware/utils.py b/lensless/hardware/utils.py index bf97cdbb..3ee77b54 100644 --- a/lensless/hardware/utils.py +++ b/lensless/hardware/utils.py @@ -21,12 +21,13 @@ def check_capture_config(config): sensor = config.sensor nbits_capture = config.nbits_capture + nbits_out = config.nbits_out exp = config.exp rgb = config.rgb gray = config.gray legacy = config.legacy down = config.down - res = config.res + res = config.res if "res" in config.keys() else None awb_gains = config.awb_gains assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" @@ -42,6 +43,8 @@ def check_capture_config(config): nbits_capture in supported_bit_depth ), f"nbits_capture must be one of {supported_bit_depth} for sensor {sensor}" + assert nbits_capture >= nbits_out, "nbits_capture must be greater than or equal to nbits_out" + if SensorParam.BLACK_LEVEL in sensor_dict[sensor]: black_level = sensor_dict[sensor][SensorParam.BLACK_LEVEL] * (2**nbits_capture - 1) else: diff --git a/lensless/utils/io.py b/lensless/utils/io.py index 750b0e0e..e36c5686 100644 --- a/lensless/utils/io.py +++ b/lensless/utils/io.py @@ -39,6 +39,10 @@ def load_image( """ Load image as numpy array. + Note that for bayer data input, the image is converted to RGB using + color correction matrix (CCM) and black level subtraction parameters + for the Raspberry Pi HQ camera (12-bit). + Parameters ---------- fp : str @@ -122,12 +126,16 @@ def load_image( if bayer: assert len(img.shape) == 2, img.shape + + # assume RPi HQ camera if nbits is None: if img.max() > 255: - # HQ camera nbits = 12 else: nbits = 8 + # convert black level from 12 bit to 8 bit + if black_level > 255: + black_level = black_level / (2**12 - 1) * 255 if back: back_img = cv2.imread(back, cv2.IMREAD_UNCHANGED) diff --git a/scripts/measure/on_device_capture.py b/scripts/measure/on_device_capture.py index d03e6e3f..b6641e3c 100644 --- a/scripts/measure/on_device_capture.py +++ b/scripts/measure/on_device_capture.py @@ -33,7 +33,6 @@ from PIL import Image from lensless.hardware.utils import get_distro, check_capture_config from lensless.utils.image import bayer2rgb_cc, rgb2gray, resize -from lensless.hardware.sensor import SensorOptions, sensor_dict, SensorParam from fractions import Fraction import time diff --git a/scripts/measure/remote_capture.py b/scripts/measure/remote_capture.py index 6777347b..be0d3d68 100644 --- a/scripts/measure/remote_capture.py +++ b/scripts/measure/remote_capture.py @@ -33,8 +33,7 @@ from pprint import pprint import matplotlib.pyplot as plt import rawpy -from lensless.hardware.utils import check_username_hostname -from lensless.hardware.sensor import SensorOptions, sensor_dict, SensorParam +from lensless.hardware.utils import check_username_hostname, check_capture_config from lensless.utils.image import rgb2gray, print_image_info from lensless.utils.plot import plot_image, pixel_histogram from lensless.utils.io import save_image @@ -44,30 +43,21 @@ @hydra.main(version_base=None, config_path="../../configs", config_name="demo") def liveview(config): - sensor = config.capture.sensor - assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" + username = config.rpi.username + hostname = config.rpi.hostname + check_username_hostname(username, hostname) + black_level, ccm, _ = check_capture_config(config.capture) + sensor = config.capture.sensor bayer = config.capture.bayer rgb = config.capture.rgb gray = config.capture.gray - - check_username_hostname(config.rpi.username, config.rpi.hostname) - username = config.rpi.username - hostname = config.rpi.hostname legacy = config.capture.legacy nbits_out = config.capture.nbits_out fn = config.capture.raw_data_fn gamma = config.capture.gamma - source = config.capture.source plot = config.plot - assert ( - nbits_out in sensor_dict[sensor][SensorParam.BIT_DEPTH] - ), f"capture.nbits_out must be one of {sensor_dict[sensor][SensorParam.BIT_DEPTH]} for sensor {sensor}" - assert ( - config.capture.nbits in sensor_dict[sensor][SensorParam.BIT_DEPTH] - ), f"capture.nbits must be one of {sensor_dict[sensor][SensorParam.BIT_DEPTH]} for sensor {sensor}" - if config.save: if config.output is not None: # make sure output directory exists @@ -84,10 +74,8 @@ def liveview(config): pic_command = ( f"{config.rpi.python} {config.capture.script} sensor={sensor} bayer={bayer} fn={remote_fn} exp={config.capture.exp} iso={config.capture.iso} " f"config_pause={config.capture.config_pause} sensor_mode={config.capture.sensor_mode} nbits_out={config.capture.nbits_out} " - f"legacy={config.capture.legacy} rgb={config.capture.rgb} gray={config.capture.gray} " + f"legacy={config.capture.legacy} rgb={config.capture.rgb} gray={config.capture.gray} nbits_capture={config.capture.nbits_capture} " ) - if config.capture.nbits > 8: - pic_command += " sixteen=True" if config.capture.down: pic_command += f" down={config.capture.down}" if config.capture.awb_gains: @@ -232,6 +220,8 @@ def liveview(config): blue_gain=blue_gain, red_gain=red_gain, nbits_out=nbits_out, + black_level=black_level, + ccm=ccm, ) # write RGB data @@ -250,22 +240,12 @@ def liveview(config): if save: plt.savefig(os.path.join(save, f"{fn}_plot.png")) - # plot red channel - if source == "red": - img_1chan = img[:, :, 0] - elif source == "green": - img_1chan = img[:, :, 1] - elif source == "blue": - img_1chan = img[:, :, 2] - else: - img_1chan = rgb2gray(img[None, :, :, :]) + # plot grayscale + img_1chan = rgb2gray(img[None, :, :, :]) ax = plot_image(img_1chan) - if source == "white": - ax.set_title("Gray scale") - else: - ax.set_title(f"{source} channel") + ax.set_title("Grayscale") if save: - plt.savefig(os.path.join(save, f"{fn}_1chan.png")) + plt.savefig(os.path.join(save, f"{fn}_gray.png")) # plot histogram, useful for checking clipping pixel_histogram(img) @@ -273,7 +253,7 @@ def liveview(config): plt.savefig(os.path.join(save, f"{fn}_hist.png")) pixel_histogram(img_1chan) if save: - plt.savefig(os.path.join(save, f"{fn}_1chan_hist.png")) + plt.savefig(os.path.join(save, f"{fn}_gray_hist.png")) else: ax = plot_image(img, gamma=gamma)