camerad: abstract out ISP handling (commaai#33583)

* spectra and tici

* master

* move the rest

* rm stupid indirection

* start move

* multi cam state is dead

* rest is moved

* lil more

* mv that

* lil more

* fix pc build

* we haven't done rgb for a while

* bring this stuff back

* fix mac?

* no camera in ui!

* i remember why we always cut mac

* fix mac build

---------

Co-authored-by: Comma Device <[email protected]>

selfdrive/ui/qt/widgets/cameraview.h

@@ -23,11 +23,9 @@
 #endif
 
 #include "msgq/visionipc/visionipc_client.h"
-#include "system/camerad/cameras/camera_common.h"
 #include "selfdrive/ui/ui.h"
 
 const int FRAME_BUFFER_SIZE = 5;
-static_assert(FRAME_BUFFER_SIZE <= YUV_BUFFER_COUNT);
 
 class CameraWidget : public QOpenGLWidget, protected QOpenGLFunctions {
   Q_OBJECT

system/camerad/SConscript

@@ -3,7 +3,7 @@ Import('env', 'arch', 'messaging', 'common', 'gpucommon', 'visionipc')
 libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', 'yuv', messaging, visionipc, gpucommon, 'atomic']
 
 camera_obj = env.Object(['cameras/camera_qcom2.cc', 'cameras/camera_common.cc', 'cameras/camera_util.cc',
-                         'sensors/ar0231.cc', 'sensors/ox03c10.cc', 'sensors/os04c10.cc'])
+                         'cameras/spectra.cc', 'sensors/ar0231.cc', 'sensors/ox03c10.cc', 'sensors/os04c10.cc'])
 env.Program('camerad', ['main.cc', camera_obj], LIBS=libs)
 
 if GetOption("extras") and arch == "x86_64":

system/camerad/cameras/camera_common.cc

@@ -10,24 +10,21 @@
 #include "common/swaglog.h"
 #include "third_party/linux/include/msm_media_info.h"
 
-#include "system/camerad/cameras/camera_qcom2.h"
-#ifdef QCOM2
-#include "CL/cl_ext_qcom.h"
-#endif
+#include "system/camerad/cameras/spectra.h"
+
 
 class ImgProc {
 public:
-  ImgProc(cl_device_id device_id, cl_context context, const CameraBuf *b, const CameraState *s, int buf_width, int uv_offset) {
+  ImgProc(cl_device_id device_id, cl_context context, const CameraBuf *b, const SensorInfo *sensor, int camera_num, int buf_width, int uv_offset) {
     char args[4096];
-    const SensorInfo *sensor = s->sensor.get();
     snprintf(args, sizeof(args),
              "-cl-fast-relaxed-math -cl-denorms-are-zero -Isensors "
              "-DFRAME_WIDTH=%d -DFRAME_HEIGHT=%d -DFRAME_STRIDE=%d -DFRAME_OFFSET=%d "
              "-DRGB_WIDTH=%d -DRGB_HEIGHT=%d -DYUV_STRIDE=%d -DUV_OFFSET=%d "
              "-DSENSOR_ID=%hu -DHDR_OFFSET=%d -DVIGNETTING=%d ",
              sensor->frame_width, sensor->frame_height, sensor->hdr_offset > 0 ? sensor->frame_stride * 2 : sensor->frame_stride, sensor->frame_offset,
              b->out_img_width, b->out_img_height, buf_width, uv_offset,
-             static_cast<unsigned short>(sensor->image_sensor), sensor->hdr_offset, s->cc.camera_num == 1);
+             static_cast<unsigned short>(sensor->image_sensor), sensor->hdr_offset, camera_num == 1);
     const char *cl_file = "cameras/process_raw.cl";
     cl_program prg_imgproc = cl_program_from_file(context, device_id, cl_file, args);
     krnl_ = CL_CHECK_ERR(clCreateKernel(prg_imgproc, "process_raw", &err));
@@ -62,12 +59,13 @@ class ImgProc {
   cl_command_queue queue;
 };
 
-void CameraBuf::init(cl_device_id device_id, cl_context context, CameraState *s, VisionIpcServer * v, int frame_cnt, VisionStreamType type) {
+void CameraBuf::init(cl_device_id device_id, cl_context context, SpectraCamera *cam, VisionIpcServer * v, int frame_cnt, VisionStreamType type) {
   vipc_server = v;
   stream_type = type;
   frame_buf_count = frame_cnt;
 
-  const SensorInfo *sensor = s->sensor.get();
+  const SensorInfo *sensor = cam->sensor.get();
+
   // RAW frame
   const int frame_size = (sensor->frame_height + sensor->extra_height) * sensor->frame_stride;
   camera_bufs = std::make_unique<VisionBuf[]>(frame_buf_count);
@@ -95,7 +93,7 @@ void CameraBuf::init(cl_device_id device_id, cl_context context, CameraState *s,
   vipc_server->create_buffers_with_sizes(stream_type, YUV_BUFFER_COUNT, false, out_img_width, out_img_height, nv12_size, nv12_width, nv12_uv_offset);
   LOGD("created %d YUV vipc buffers with size %dx%d", YUV_BUFFER_COUNT, nv12_width, nv12_height);
 
-  imgproc = new ImgProc(device_id, context, this, s, nv12_width, nv12_uv_offset);
+  imgproc = new ImgProc(device_id, context, this, sensor, cam->cc.camera_num, nv12_width, nv12_uv_offset);
 }
 
 CameraBuf::~CameraBuf() {
@@ -138,24 +136,6 @@ void CameraBuf::queue(size_t buf_idx) {
 
 // common functions
 
-void fill_frame_data(cereal::FrameData::Builder &framed, const FrameMetadata &frame_data, CameraState *c) {
-  framed.setFrameId(frame_data.frame_id);
-  framed.setRequestId(frame_data.request_id);
-  framed.setTimestampEof(frame_data.timestamp_eof);
-  framed.setTimestampSof(frame_data.timestamp_sof);
-  framed.setIntegLines(frame_data.integ_lines);
-  framed.setGain(frame_data.gain);
-  framed.setHighConversionGain(frame_data.high_conversion_gain);
-  framed.setMeasuredGreyFraction(frame_data.measured_grey_fraction);
-  framed.setTargetGreyFraction(frame_data.target_grey_fraction);
-  framed.setProcessingTime(frame_data.processing_time);
-
-  const float ev = c->cur_ev[frame_data.frame_id % 3];
-  const float perc = util::map_val(ev, c->sensor->min_ev, c->sensor->max_ev, 0.0f, 100.0f);
-  framed.setExposureValPercent(perc);
-  framed.setSensor(c->sensor->image_sensor);
-}
-
 kj::Array<uint8_t> get_raw_frame_image(const CameraBuf *b) {
   const uint8_t *dat = (const uint8_t *)b->cur_camera_buf->addr;
 
@@ -283,30 +263,6 @@ float set_exposure_target(const CameraBuf *b, Rect ae_xywh, int x_skip, int y_sk
   return lum_med / 256.0;
 }
 
-void camerad_thread() {
-  cl_device_id device_id = cl_get_device_id(CL_DEVICE_TYPE_DEFAULT);
-#ifdef QCOM2
-  const cl_context_properties props[] = {CL_CONTEXT_PRIORITY_HINT_QCOM, CL_PRIORITY_HINT_HIGH_QCOM, 0};
-  cl_context context = CL_CHECK_ERR(clCreateContext(props, 1, &device_id, NULL, NULL, &err));
-#else
-  cl_context context = CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
-#endif
-
-  {
-    MultiCameraState cameras;
-    cameras.pm = new PubMaster({"roadCameraState", "driverCameraState", "wideRoadCameraState", "thumbnail"});
-    VisionIpcServer vipc_server("camerad", device_id, context);
-
-    cameras_init(&cameras, &vipc_server, device_id, context);
-
-    vipc_server.start_listener();
-
-    cameras_run(&cameras);
-  }
-
-  CL_CHECK(clReleaseContext(context));
-}
-
 int open_v4l_by_name_and_index(const char name[], int index, int flags) {
   for (int v4l_index = 0; /**/; ++v4l_index) {
     std::string v4l_name = util::read_file(util::string_format("/sys/class/video4linux/v4l-subdev%d/name", v4l_index));

system/camerad/cameras/camera_common.h

@@ -8,6 +8,7 @@
 #include "common/queue.h"
 #include "common/util.h"
 
+
 const int YUV_BUFFER_COUNT = 20;
 
 enum CameraType {
@@ -39,7 +40,7 @@ typedef struct FrameMetadata {
   float processing_time;
 } FrameMetadata;
 
-struct MultiCameraState;
+class SpectraCamera;
 class CameraState;
 class ImgProc;
 
@@ -62,18 +63,13 @@ class CameraBuf {
 
   CameraBuf() = default;
   ~CameraBuf();
-  void init(cl_device_id device_id, cl_context context, CameraState *s, VisionIpcServer * v, int frame_cnt, VisionStreamType type);
+  void init(cl_device_id device_id, cl_context context, SpectraCamera *cam, VisionIpcServer * v, int frame_cnt, VisionStreamType type);
   bool acquire();
   void queue(size_t buf_idx);
 };
 
-void fill_frame_data(cereal::FrameData::Builder &framed, const FrameMetadata &frame_data, CameraState *c);
+void camerad_thread();
 kj::Array<uint8_t> get_raw_frame_image(const CameraBuf *b);
 float set_exposure_target(const CameraBuf *b, Rect ae_xywh, int x_skip, int y_skip);
 void publish_thumbnail(PubMaster *pm, const CameraBuf *b);
-void cameras_init(MultiCameraState *s, VisionIpcServer *v, cl_device_id device_id, cl_context ctx);
-void cameras_run(MultiCameraState *s);
-void cameras_close(MultiCameraState *s);
-void camerad_thread();
-
 int open_v4l_by_name_and_index(const char name[], int index = 0, int flags = O_RDWR | O_NONBLOCK);