Create JNI

CMakeLists.txt

@@ -12,6 +12,30 @@ find_package(OpenCV REQUIRED)
 pkg_check_modules(LIBDRM REQUIRED libdrm)
 pkg_check_modules(LIBCAMERA REQUIRED libcamera)
 
-add_executable(libcamera_meme main.cpp concurrent_blocking_queue.h camera_grabber.cpp dma_buf_alloc.cpp gl_hsv_thresholder.cpp libcamera_opengl_utility.cpp)
-target_include_directories(libcamera_meme PUBLIC ${OPENGL_INCLUDE_DIRS} ${LIBDRM_INCLUDE_DIRS} ${LIBCAMERA_INCLUDE_DIRS} ${OpenCV_INCLUDE_DIRS})
+find_package(JNI)
+if (JNI_FOUND)
+    # Fixes odd AWT dependency
+    set (JNI_INCLUDE_DIRS ${JAVA_INCLUDE_PATH} ${JAVA_INCLUDE_PATH2})
+    message (STATUS "JNI_INCLUDE_DIRS=${JNI_INCLUDE_DIRS}")
+endif()
+
+add_executable(libcamera_meme 
+    main.cpp 
+    concurrent_blocking_queue.h 
+    camera_grabber.cpp 
+    dma_buf_alloc.cpp 
+    gl_hsv_thresholder.cpp 
+    libcamera_opengl_utility.cpp
+    libcamera_jni.hpp
+    libcamera_jni.cpp
+)
+
+    target_include_directories(libcamera_meme PUBLIC
+    ${OPENGL_INCLUDE_DIRS} 
+    ${LIBDRM_INCLUDE_DIRS}
+    ${LIBCAMERA_INCLUDE_DIRS}
+    ${OpenCV_INCLUDE_DIRS}
+    ${JNI_INCLUDE_DIRS}
+)
+
 target_link_libraries(libcamera_meme PUBLIC OpenGL::GL OpenGL::EGL Threads::Threads ${LIBCAMERA_LINK_LIBRARIES} ${OpenCV_LIBS})

LibCameraJNI.java

@@ -0,0 +1,24 @@
+public class LibCameraJNI {
+    public static native String getSensorModelRaw();
+    public static native boolean isSupported();
+    // Get a pointer to a new runner
+    public static native long createCamera(int width, int height);
+    public static native boolean destroyCamera(long ptr);
+
+    // Set thresholds on [0..1]
+    public static native boolean setThresholds(long ptr,
+        double hl, double sl, double vl,
+        double hu, double su, double vu);
+
+    // Exposure time, in microseconds
+    public static native boolean setExposure(long ptr, int exposureUs);
+    public static native boolean setBrightness(long ptr, double brightness);
+    public static native boolean setAwbGain(long ptr, float red, float blue);
+    public static native boolean setAnalogGain(long ptr, double analog);
+    public static native boolean setDigitalGain(long ptr, double digital);
+    public static native boolean setShouldGreyscale(long ptr, boolean shouldGreyscale);
+
+    public static native boolean awaitNewFrame(long ptr);
+    public static native long getColorFrame(long ptr);
+    public static native long getGPUoutput(long ptr);
+}

camera_grabber.cpp

@@ -72,19 +72,41 @@ void CameraGrabber::requestComplete(libcamera::Request *request) {
 }
 
 void CameraGrabber::requeueRequest(libcamera::Request *request) {
+    // This resets all our controls
+    // https://github.com/kbingham/libcamera/blob/master/src/libcamera/request.cpp#L397
     request->reuse(libcamera::Request::ReuseFlag::ReuseBuffers);
+
+    setControls(request);
+
     if (m_camera->queueRequest(request) < 0) {
         throw std::runtime_error("failed to queue request");
     }
 }
 
+void CameraGrabber::setControls(libcamera::Request *request) {
+    auto &controls = request->controls();
+    controls.set(libcamera::controls::AeEnable, false); // Auto exposure disabled
+    controls.set(libcamera::controls::AwbEnable, false); // AWB disabled
+    controls.set(libcamera::controls::ExposureTime, m_settings.exposureTimeUs); // in microseconds
+    controls.set(libcamera::controls::AnalogueGain, m_settings.analogGain); // Analog gain, min 1 max big number?
+    controls.set(libcamera::controls::DigitalGain, m_settings.digitalGain); // Digital gain, unknown range
+    controls.set(libcamera::controls::ColourGains, {m_settings.awbRedGain, m_settings.awbBlueGain}); // AWB gains, red and blue, unknown range
+    controls.set(libcamera::controls::Brightness, m_settings.brightness); // -1 to 1, 0 means unchanged
+    controls.set(libcamera::controls::Contrast, m_settings.contrast); // Nominal 1
+    controls.set(libcamera::controls::Saturation, m_settings.saturation); // Nominal 1, 0 would be greyscale
+    controls.set(libcamera::controls::FrameDurationLimits, {static_cast<int64_t>(0), static_cast<int64_t>(0)}); // Set default to zero, we have specificed the exposure time
+
+    // Additionally, we can set crop regions and stuff
+}
+
 void CameraGrabber::startAndQueue() {
     if (m_camera->start()) {
         throw std::runtime_error("failed to start camera");
     }
 
 // TODO: HANDLE THIS BETTER
     for (auto &request: m_requests) {
+        setControls(request);
         if (m_camera->queueRequest(request.get()) < 0) {
             throw std::runtime_error("failed to queue request");
         }

camera_grabber.h

@@ -8,6 +8,17 @@
 #include <functional>
 #include <optional>
 
+struct CameraSettings {
+    int32_t exposureTimeUs;
+    float analogGain;
+    float digitalGain
+    float brightness;
+    float contrast;
+    float awbRedGain;
+    float awbBlueGain;
+    float saturation;
+}
+
 class CameraGrabber {
 public:
     explicit CameraGrabber(std::shared_ptr<libcamera::Camera> camera, int width, int height);
@@ -18,11 +29,17 @@ class CameraGrabber {
     void startAndQueue();
     void requeueRequest(libcamera::Request *request);
 
+    CameraSettings& GetCameraSettings() { return m_settings; }
+
 private:
+    void setControls(std::function<void(libcamera::Request*)> request);
+
     std::vector<std::unique_ptr<libcamera::Request>> m_requests;
     std::map<int, const char *> m_mapped;
     void requestComplete(libcamera::Request *request);
 
+    std::vector<std::pair<int, libcamera::ControlValue>> m_controlUpdatesToApply;
+    CameraSettings m_settings;
     std::shared_ptr<libcamera::Camera> m_camera;
     std::unique_ptr<libcamera::CameraConfiguration> m_config;
     libcamera::FrameBufferAllocator m_buf_allocator;

camera_manager.cpp

@@ -0,0 +1,123 @@
+#include "camera_manager.h"
+
+CameraRunner::CameraRunner(int width, int height, const std::string& id) {
+    auto cameras = camera_manager_->cameras();
+    auto *camera = std::find_if(cameras.begin(), cameras.end(), [](auto &cam) { return cam->id() == id; })
+
+    auto allocer = DmaBufAlloc("/dev/dma_heap/linux,cma");
+
+    auto grabber = CameraGrabber(std::move(camera), width, height);
+    unsigned int stride = grabber.streamConfiguration().stride;
+
+    auto camera_queue = ConcurrentBlockingQueue<libcamera::Request *>();
+    grabber.setOnData([&](libcamera::Request *request) {
+        camera_queue.push(request);
+    });
+
+    std::vector<int> fds {
+            allocer.alloc_buf(width * height * 4),
+            allocer.alloc_buf(width * height * 4),
+            allocer.alloc_buf(width * height * 4)
+    };
+
+
+    m_threshold ([&]() {
+        auto colorspace = grabber.streamConfiguration().colorSpace.value();
+        auto thresholder = GlHsvThresholder(width, height, fds);
+
+        auto gpu_queue = ConcurrentBlockingQueue<int>();
+        thresholder.setOnComplete([&](int fd) {
+            gpu_queue.push(fd);
+        });
+
+
+        std::thread display([&]() {
+            std::unordered_map<int, unsigned char *> mmaped;
+
+            for (auto fd: fds) {
+                auto mmap_ptr = mmap(nullptr, width * height * 4, PROT_READ, MAP_SHARED, fd, 0);
+                if (mmap_ptr == MAP_FAILED) {
+                    throw std::runtime_error("failed to mmap pointer");
+                }
+                mmaped.emplace(fd, static_cast<unsigned char *>(mmap_ptr));
+            }
+
+            cv::Mat threshold_mat(height, width, CV_8UC1);
+            unsigned char *threshold_out_buf = threshold_mat.data;
+            cv::Mat color_mat(height, width, CV_8UC3);
+            unsigned char *color_out_buf = color_mat.data;
+
+            while (display_thread_run) {
+                auto fd = gpu_queue.pop();
+                if (fd == -1) {
+                    break;
+                }
+
+                auto input_ptr = mmaped.at(fd);
+                int bound = width * height;
+
+                for (int i = 0; i < bound; i++) {
+                    std::memcpy(color_out_buf + i * 3, input_ptr + i * 4, 3);
+                    threshold_out_buf[i] = input_ptr[i * 4 + 3];
+                }
+
+                // pls don't optimize these writes out compiler
+                std::cout << reinterpret_cast<uint64_t>(threshold_out_buf) << " " << reinterpret_cast<uint64_t>(color_out_buf) << std::endl;
+
+                thresholder.returnBuffer(fd);
+                // cv::imshow("cam", mat);
+                // cv::waitKey(3);
+            }
+        });
+
+        while (threshold_thread_run) {
+            auto request = camera_queue.pop();
+
+            if (!request) {
+                break;
+            }
+
+            // in Nanoseconds, from the Linux CLOCK_BOOTTIME 
+            auto sensorTimestamp = request->controls()->get(libcamera::controls::SensorTimestamp); 
+
+            auto planes = request->buffers().at(grabber.streamConfiguration().stream())->planes();
+
+            std::array<GlHsvThresholder::DmaBufPlaneData, 3> yuv_data {{
+             {planes[0].fd.get(),
+              static_cast<EGLint>(planes[0].offset),
+              static_cast<EGLint>(stride)},
+             {planes[1].fd.get(),
+              static_cast<EGLint>(planes[1].offset),
+              static_cast<EGLint>(stride / 2)},
+             {planes[2].fd.get(),
+              static_cast<EGLint>(planes[2].offset),
+              static_cast<EGLint>(stride / 2)},
+             }};
+
+            thresholder.testFrame(yuv_data, encodingFromColorspace(colorspace), rangeFromColorspace(colorspace));
+            grabber.requeueRequest(request);
+        }
+
+        // need to stop above thread too
+        display_thread_run = false;
+        display.join();
+    });
+
+    std::this_thread::sleep_for(std::chrono::seconds(2));
+
+    grabber.startAndQueue();
+}
+
+void CameraRunner::Stop() {
+  threshold_thread_run = false;
+  m_threshold.join();
+}
+
+static std::vector<std::string> GetAllCameraIDs() {
+  static libcamera::CameraManager* camera_manager;
+  if(!camera_manager) {
+    camera_manager = std::make_unique<libcamera::CameraManager>();
+    camera_manager->start();
+  }
+  return camera_manager->cameras();
+}

camera_manager.h

@@ -0,0 +1,34 @@
+#include <libcamera/camera.h>
+#include <libcamera/camera_manager.h>
+
+#include <thread>
+#include <chrono>
+#include <iostream>
+#include <cstring>
+
+#include <opencv2/core.hpp>
+#include <opencv2/highgui.hpp>
+#include <sys/mman.h>
+
+#include "dma_buf_alloc.h"
+#include "gl_hsv_thresholder.h"
+#include "camera_grabber.h"
+#include "libcamera_opengl_utility.h"
+#include "concurrent_blocking_queue.h"
+
+class CameraRunner {
+public:
+  CameraRunner(int width, int height, int fps, const std::string &id);
+
+  CameraGrabber GetCameraGrabber();
+
+  void Stop();
+
+private:
+  std::thread m_threshold;
+
+  bool threshold_thread_run = true;
+  bool display_thread_run = true;
+}
+
+static std::vector<std::string> GetAllCameraIDs();

gl_hsv_thresholder.cpp

@@ -369,12 +369,13 @@ void GlHsvThresholder::testFrame(const std::array<GlHsvThresholder::DmaBufPlaneD
     glEnableVertexAttribArray(attr_loc);
     GLERROR();
     glVertexAttribPointer(attr_loc, 2, GL_FLOAT, GL_FALSE, 0, nullptr);
+
     GLERROR();
     static auto lll = glGetUniformLocation(m_program, "lowerThresh");
-    glUniform3f(lll, 0.0, 50.0 / 255.0, 50.0 / 255.0);
+    glUniform3f(lll, m_hsvLower[0], m_hsvLower[1], m_hsvLower[3]);
     GLERROR();
     static auto uuu = glGetUniformLocation(m_program, "upperThresh");
-    glUniform3f(uuu, 1.0, 1.0, 1.0);
+    glUniform3f(uuu, m_hsvUpper[0], m_hsvUpper[1], m_hsvUpper[3]);
     GLERROR();
 
     glDrawArrays(GL_TRIANGLES, 0, 6);
@@ -400,3 +401,13 @@ void GlHsvThresholder::returnBuffer(int fd) {
     std::scoped_lock lock(m_renderable_mutex);
     m_renderable.push(fd);
 }
+
+void GlHsvThresholder::setHsvThresholds(double hl, double sl, double vl, double hu, double su, double vu) {
+    m_hsvLower[0] = hl;
+    m_hsvLower[1] = sl;
+    m_hsvLower[2] = vl;
+
+    m_hsvUpper[0] = hu;
+    m_hsvUpper[1] = su;
+    m_hsvUpper[2] = vu;
+}

gl_hsv_thresholder.h

@@ -27,6 +27,19 @@ class GlHsvThresholder {
 
     void returnBuffer(int fd);
     void testFrame(const std::array<GlHsvThresholder::DmaBufPlaneData, 3>& yuv_plane_data, EGLint encoding, EGLint range);
+
+    /**
+     * @brief Set the Hsv Thresholds range, on [0..1]
+     * 
+     * @param hl Lower hue
+     * @param sl Sat lower
+     * @param vl Value lower
+     * @param hu Upper hue
+     * @param su Sat Upper
+     * @param vu Value Upper
+     */
+    void setHsvThresholds(double hl, double sl, double vl, double hu, double su, double vu);
+
 private:
     int m_width;
     int m_height;
@@ -42,6 +55,9 @@ class GlHsvThresholder {
 
     GLuint m_quad_vbo;
     GLuint m_program;
+
+    double m_hsvLower[3] = {0}; // Hue, sat, value, in [0,1]
+    double m_hsvUpper[3] = {0}; // Hue, sat, value, in [0,1]
 };
 
 #endif //LIBCAMERA_MEME_GL_HSV_THRESHOLDER_H