diff --git a/configs/capture.yaml b/configs/capture.yaml index f82c21f5..18b6c030 100644 --- a/configs/capture.yaml +++ b/configs/capture.yaml @@ -1,3 +1,4 @@ +sensor: rpi_hq bayer: True fn: test exp: 0.5 @@ -9,7 +10,7 @@ sensor_mode: "0" rgb: False gray: False iso: 100 -sixteen: True # whether 16 bits or 8 +sixteen: True # whether 16 bits or 8 (from Bayer data) legacy: True down: null res: null diff --git a/configs/demo.yaml b/configs/demo.yaml index ddc0c528..47f13ba4 100644 --- a/configs/demo.yaml +++ b/configs/demo.yaml @@ -30,6 +30,7 @@ display: white: False capture: + sensor: rpi_hq gamma: null # for visualization exp: 0.02 delay: 2 @@ -41,7 +42,8 @@ capture: nbits_out: 8 # light data transer, doesn't seem to worsen performance nbits: 12 legacy: True - gray: False + gray: False # only for legacy=True, if bayer=True, remote script returns grayscale data + # rgb: False # only for legacy=True, if bayer=True, remote script return RGB data raw_data_fn: raw_data bayer: True source: white diff --git a/configs/remote_capture_rpi_gs.yaml b/configs/remote_capture_rpi_gs.yaml new file mode 100644 index 00000000..0466e689 --- /dev/null +++ b/configs/remote_capture_rpi_gs.yaml @@ -0,0 +1,23 @@ +# python scripts/measure/remote_capture.py -cn remote_capture_rpi_gs +defaults: + - demo + - _self_ + +output: rpi_gs_capture # output folder for results +save: True +plot: True + +rpi: + username: null + hostname: null + python: ~/LenslessPiCam/lensless_env/bin/python + +capture: + sensor: rpi_gs + exp: 0.2 + bayer: True + legacy: False # must be False for rpi_gs + rgb: False + gray: False + down: null + awb_gains: null diff --git a/lensless/hardware/sensor.py b/lensless/hardware/sensor.py index 08a00a05..c842dc2d 100644 --- a/lensless/hardware/sensor.py +++ b/lensless/hardware/sensor.py @@ -58,10 +58,14 @@ class SensorParam: DIAGONAL = "diagonal" COLOR = "color" BIT_DEPTH = "bit_depth" + MAX_EXPOSURE = "max_exposure" # in seconds + MIN_EXPOSURE = "min_exposure" # in seconds """ Note sensors are in landscape orientation. + +Max exposure for RPi cameras: https://www.raspberrypi.com/documentation/accessories/camera.html#hardware-specification """ sensor_dict = { # Raspberry Pi HQ Camera Sensor @@ -73,6 +77,8 @@ class SensorParam: SensorParam.DIAGONAL: 7.857e-3, SensorParam.COLOR: True, SensorParam.BIT_DEPTH: [8, 12], + SensorParam.MAX_EXPOSURE: 670.74, + SensorParam.MIN_EXPOSURE: 0.02, }, # Raspberry Pi Global Shutter Camera # https://www.raspberrypi.com/products/raspberry-pi-global-shutter-camera/ @@ -83,6 +89,8 @@ class SensorParam: SensorParam.DIAGONAL: 6.3e-3, SensorParam.COLOR: True, SensorParam.BIT_DEPTH: [8, 12], + SensorParam.MAX_EXPOSURE: 15534385e-6, + SensorParam.MIN_EXPOSURE: 29e-6, }, # Raspberry Pi Camera Module V2 # https://www.raspberrypi.com/documentation/accessories/camera.html#hardware-specification @@ -93,6 +101,8 @@ class SensorParam: SensorParam.DIAGONAL: 4.6e-3, SensorParam.COLOR: True, SensorParam.BIT_DEPTH: [8], + SensorParam.MAX_EXPOSURE: 11.76, + SensorParam.MIN_EXPOSURE: 0.02, # TODO : verify }, # Basler daA720-520um # https://www.baslerweb.com/en/products/cameras/area-scan-cameras/dart/daa720-520um-cs-mount/ @@ -125,7 +135,14 @@ class VirtualSensor(object): """ def __init__( - self, pixel_size, resolution, diagonal=None, color=True, bit_depth=None, downsample=None + self, + pixel_size, + resolution, + diagonal=None, + color=True, + bit_depth=None, + downsample=None, + **kwargs, ): """ Base constructor. diff --git a/lensless/hardware/utils.py b/lensless/hardware/utils.py index 97b384f6..a52668c5 100644 --- a/lensless/hardware/utils.py +++ b/lensless/hardware/utils.py @@ -58,6 +58,9 @@ def display( def check_username_hostname(username, hostname, timeout=10): + assert username is not None, "Username must be specified" + assert hostname is not None, "Hostname must be specified" + client = paramiko.client.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) diff --git a/scripts/measure/on_device_capture.py b/scripts/measure/on_device_capture.py index 22241807..f0aae9b5 100644 --- a/scripts/measure/on_device_capture.py +++ b/scripts/measure/on_device_capture.py @@ -2,7 +2,20 @@ Capture raw Bayer data or post-processed RGB data. ``` -python scripts/on_device_capture.py --legacy --exp 0.02 --sensor_mode 0 +python scripts/measure/on_device_capture.py legacy=True \ +exp=0.02 bayer=True +``` + +With the Global Shutter sensor, legacy RPi software is not supported. +``` +python scripts/measure/on_device_capture.py sensor=rpi_gs \ +legacy=False exp=0.02 bayer=True +``` + +To capture PNG data (bayer=False) and downsample (by factor 2): +``` +python scripts/measure/on_device_capture.py sensor=rpi_gs \ +legacy=False exp=0.02 bayer=False down=2 ``` See these code snippets for setting camera settings and post-processing @@ -21,6 +34,7 @@ from lensless.hardware.utils import get_distro from lensless.utils.image import bayer2rgb_cc, rgb2gray, resize from lensless.hardware.constants import RPI_HQ_CAMERA_CCM_MATRIX, RPI_HQ_CAMERA_BLACK_LEVEL +from lensless.hardware.sensor import SensorOptions, sensor_dict, SensorParam from fractions import Fraction import time @@ -42,6 +56,9 @@ @hydra.main(version_base=None, config_path="../../configs", config_name="capture") def capture(config): + sensor = config.sensor + assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" + bayer = config.bayer fn = config.fn exp = config.exp @@ -56,47 +73,99 @@ def capture(config): res = config.res nbits_out = config.nbits_out - # https://www.raspberrypi.com/documentation/accessories/camera.html#maximum-exposure-times - # TODO : check which camera - assert exp <= 230 - assert exp >= 0.02 + # https://www.raspberrypi.com/documentation/accessories/camera.html#hardware-specification + sensor_param = sensor_dict[sensor] + assert exp <= sensor_param[SensorParam.MAX_EXPOSURE] + assert exp >= sensor_param[SensorParam.MIN_EXPOSURE] sensor_mode = int(sensor_mode) distro = get_distro() print("RPi distribution : {}".format(distro)) + + if sensor == SensorOptions.RPI_GS.value: + assert not legacy + if "bullseye" in distro and not legacy: # TODO : grayscale and downsample assert not rgb assert not gray - assert down is None import subprocess - jpg_fn = fn + ".jpg" - dng_fn = fn + ".dng" - pic_command = [ - "libcamera-still", - "-r", - "--gain", - f"{iso / 100}", - "--shutter", - f"{int(exp * 1e6)}", - "-o", - f"{jpg_fn}", - ] - - cmd = subprocess.Popen( - pic_command, - shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - cmd.stdout.readlines() - cmd.stderr.readlines() - os.remove(jpg_fn) - os.system(f"exiftool {dng_fn}") - print("\nJPG saved to : {}".format(jpg_fn)) - print("\nDNG saved to : {}".format(dng_fn)) + if bayer: + + assert down is None + + jpg_fn = fn + ".jpg" + fn += ".dng" + pic_command = [ + "libcamera-still", + "-r", + "--gain", + f"{iso / 100}", + "--shutter", + f"{int(exp * 1e6)}", + "-o", + f"{jpg_fn}", + ] + + cmd = subprocess.Popen( + pic_command, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + cmd.stdout.readlines() + cmd.stderr.readlines() + # os.remove(jpg_fn) + os.system(f"exiftool {fn}") + print("\nJPG saved to : {}".format(jpg_fn)) + # print("\nDNG saved to : {}".format(fn)) + + else: + + from picamera2 import Picamera2, Preview + + picam2 = Picamera2() + picam2.start_preview(Preview.NULL) + + fn += ".png" + + max_res = picam2.camera_properties["PixelArraySize"] + if res: + assert len(res) == 2 + else: + res = np.array(max_res) + if down is not None: + res = (np.array(res) / down).astype(int) + + res = tuple(res) + print("Capturing at resolution: ", res) + + # capture low-dim PNG + picam2.preview_configuration.main.size = res + picam2.still_configuration.size = res + picam2.still_configuration.enable_raw() + picam2.still_configuration.raw.size = res + + # setting camera parameters + picam2.configure(picam2.create_preview_configuration()) + new_controls = { + "ExposureTime": int(exp * 1e6), + "AnalogueGain": 1.0, + } + if config.awb_gains is not None: + assert len(config.awb_gains) == 2 + new_controls["ColourGains"] = tuple(config.awb_gains) + picam2.set_controls(new_controls) + + # take picture + picam2.start("preview", show_preview=False) + time.sleep(config.config_pause) + + picam2.switch_mode_and_capture_file("still", fn) + + # legacy camera software else: import picamerax.array @@ -104,36 +173,6 @@ def capture(config): if bayer: - # if rgb: - - # camera = picamerax.PiCamera(framerate=1 / exp, sensor_mode=sensor_mode, resolution=res) - # camera.iso = iso - # # Wait for the automatic gain control to settle - # sleep(config_pause) - # # Now fix the values - # camera.shutter_speed = camera.exposure_speed - # camera.exposure_mode = "off" - # g = camera.awb_gains - # camera.awb_mode = "off" - # camera.awb_gains = g - - # print("Resolution : {}".format(camera.resolution)) - # print("Shutter speed : {}".format(camera.shutter_speed)) - # print("ISO : {}".format(camera.iso)) - # print("Frame rate : {}".format(camera.framerate)) - # print("Sensor mode : {}".format(SENSOR_MODES[sensor_mode])) - # # keep this as it needs to be parsed from remote script! - # red_gain = float(g[0]) - # blue_gain = float(g[1]) - # print("Red gain : {}".format(red_gain)) - # print("Blue gain : {}".format(blue_gain)) - - # # take picture - # fn += ".png" - # camera.capture(str(fn), bayer=False, resize=None) - - # else: - camera = picamerax.PiCamera(framerate=1 / exp, sensor_mode=sensor_mode, resolution=res) # camera settings, as little processing as possible @@ -171,6 +210,7 @@ def capture(config): else: output = (np.sum(stream.array, axis=2) >> 2).astype(np.uint8) + # returning non-bayer data if rgb or gray: if sixteen: n_bits = 12 # assuming Raspberry Pi HQ @@ -209,8 +249,7 @@ def capture(config): else: - # returning non-bayer data - + # capturing and returning non-bayer data from picamerax import PiCamera camera = PiCamera() @@ -221,6 +260,9 @@ def capture(config): if down is not None: res = (np.array(res) / down).astype(int) + # -- now set up camera with desired settings + camera = PiCamera(framerate=1 / exp, sensor_mode=sensor_mode, resolution=tuple(res)) + # Wait for the automatic gain control to settle time.sleep(config.config_pause) @@ -243,7 +285,7 @@ def capture(config): "Out of resources! Use bayer for higher resolution, or increase `gpu_mem` in `/boot/config.txt`." ) - print("\nImage saved to : {}".format(fn)) + print("Image saved to : {}".format(fn)) if __name__ == "__main__": diff --git a/scripts/measure/remote_capture.py b/scripts/measure/remote_capture.py index 66210a86..411a0f4c 100644 --- a/scripts/measure/remote_capture.py +++ b/scripts/measure/remote_capture.py @@ -1,11 +1,28 @@ """ -python scripts/measure/remote_capture.py +For Bayer data with RPI HQ sensor: +``` +python scripts/measure/remote_capture.py \ +rpi.username=USERNAME rpi.hostname=IP_ADDRESS +``` + +For Bayer data with RPI Global shutter sensor: +``` +python scripts/measure/remote_capture.py -cn remote_capture_rpi_gs \ +rpi.username=USERNAME rpi.hostname=IP_ADDRESS +``` + +For RGB data with RPI HQ RPI Global shutter sensor: +``` +python scripts/measure/remote_capture.py -cn remote_capture_rpi_gs \ +rpi.username=USERNAME rpi.hostname=IP_ADDRESS \ +capture.bayer=False capture.down=2 +``` Check out the `configs/demo.yaml` file for parameters, specifically: - `rpi`: RPi parameters -- `capture`: parameters for displaying image +- `capture`: parameters for taking pictures """ @@ -17,8 +34,7 @@ import matplotlib.pyplot as plt import rawpy from lensless.hardware.utils import check_username_hostname - - +from lensless.hardware.sensor import SensorOptions from lensless.utils.image import rgb2gray, print_image_info from lensless.utils.plot import plot_image, pixel_histogram from lensless.utils.io import save_image @@ -28,6 +44,9 @@ @hydra.main(version_base=None, config_path="../../configs", config_name="demo") def liveview(config): + sensor = config.capture.sensor + assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}" + bayer = config.capture.bayer rgb = config.capture.rgb gray = config.capture.gray @@ -52,27 +71,16 @@ def liveview(config): else: save = False - # proceed with capture - # if bayer: - # assert not rgb - # assert not gray - assert hostname is not None - # take picture remote_fn = "remote_capture" print("\nTaking picture...") pic_command = ( - f"{config.rpi.python} {config.capture.script} bayer={bayer} fn={remote_fn} exp={config.capture.exp} iso={config.capture.iso} " - f"config_pause={config.capture.config_pause} sensor_mode={config.capture.sensor_mode} nbits_out={config.capture.nbits_out}" + f"{config.rpi.python} {config.capture.script} sensor={sensor} bayer={bayer} fn={remote_fn} exp={config.capture.exp} iso={config.capture.iso} " + f"config_pause={config.capture.config_pause} sensor_mode={config.capture.sensor_mode} nbits_out={config.capture.nbits_out} " + f"legacy={config.capture.legacy} rgb={config.capture.rgb} gray={config.capture.gray} " ) if config.capture.nbits > 8: pic_command += " sixteen=True" - if config.capture.rgb: - pic_command += " rgb=True" - if config.capture.legacy: - pic_command += " legacy=True" - if config.capture.gray: - pic_command += " gray=True" if config.capture.down: pic_command += f" down={config.capture.down}" if config.capture.awb_gains: @@ -88,7 +96,7 @@ def liveview(config): result = ssh.stdout.readlines() error = ssh.stderr.readlines() - if error != []: + if error != [] and legacy: # new camera software seems to return error even if it works print("ERROR: %s" % error) return if result == []: @@ -112,46 +120,59 @@ def liveview(config): and "bullseye" in result_dict["RPi distribution"] and not legacy ): - # copy over DNG file - remotefile = f"~/{remote_fn}.dng" - localfile = f"{fn}.dng" - print(f"\nCopying over picture as {localfile}...") - os.system('scp "%s@%s:%s" %s' % (username, hostname, remotefile, localfile)) - raw = rawpy.imread(localfile) - - # https://letmaik.github.io/rawpy/api/rawpy.Params.html#rawpy.Params - # https://www.libraw.org/docs/API-datastruct-eng.html - if nbits_out > 8: - # only 8 or 16 bit supported by postprocess - if nbits_out != 16: - print("casting to 16 bit...") - output_bps = 16 + + if bayer: + + # copy over DNG file + remotefile = f"~/{remote_fn}.dng" + localfile = f"{fn}.dng" + print(f"\nCopying over picture as {localfile}...") + os.system('scp "%s@%s:%s" %s' % (username, hostname, remotefile, localfile)) + raw = rawpy.imread(localfile) + + # https://letmaik.github.io/rawpy/api/rawpy.Params.html#rawpy.Params + # https://www.libraw.org/docs/API-datastruct-eng.html + if nbits_out > 8: + # only 8 or 16 bit supported by postprocess + if nbits_out != 16: + print("casting to 16 bit...") + output_bps = 16 + else: + if nbits_out != 8: + print("casting to 8 bit...") + output_bps = 8 + img = raw.postprocess( + adjust_maximum_thr=0, # default 0.75 + no_auto_scale=False, + gamma=(1, 1), + output_bps=output_bps, + bright=1, # default 1 + exp_shift=1, + no_auto_bright=True, + use_camera_wb=True, + use_auto_wb=False, # default is False? f both use_camera_wb and use_auto_wb are True, then use_auto_wb has priority. + ) + + # print image properties + print_image_info(img) + + # save as PNG + png_out = f"{fn}.png" + print(f"Saving RGB file as: {png_out}") + cv2.imwrite(png_out, cv2.cvtColor(img, cv2.COLOR_RGB2BGR)) + else: - if nbits_out != 8: - print("casting to 8 bit...") - output_bps = 8 - img = raw.postprocess( - adjust_maximum_thr=0, # default 0.75 - no_auto_scale=False, - gamma=(1, 1), - output_bps=output_bps, - bright=1, # default 1 - exp_shift=1, - no_auto_bright=True, - use_camera_wb=True, - use_auto_wb=False, # default is False? f both use_camera_wb and use_auto_wb are True, then use_auto_wb has priority. - ) - - # print image properties - print_image_info(img) - - # save as PNG - png_out = f"{fn}.png" - print(f"Saving RGB file as: {png_out}") - cv2.imwrite(png_out, cv2.cvtColor(img, cv2.COLOR_RGB2BGR)) - if not bayer: - os.remove(localfile) + remotefile = f"~/{remote_fn}.png" + localfile = f"{fn}.png" + if save: + localfile = os.path.join(save, localfile) + print(f"\nCopying over picture as {localfile}...") + os.system('scp "%s@%s:%s" %s' % (username, hostname, remotefile, localfile)) + + img = load_image(localfile, verbose=True) + + # legacy software running on RPi else: # copy over file # more pythonic? https://stackoverflow.com/questions/250283/how-to-scp-in-python