raylib cameraview

selfdrive/ui/SConscript

@@ -1,6 +1,6 @@
 import os
 import json
-Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', 'transformations')
+Import('env', 'qt_env', 'arch', 'common', 'messaging', 'visionipc', 'transformations')
 
 base_libs = [common, messaging, visionipc, transformations,
  'm', 'OpenCL', 'ssl', 'crypto', 'pthread'] + qt_env["LIBS"]
@@ -109,4 +109,14 @@ if GetOption('extras') and arch != "Darwin":
 
 # build watch3
 if arch in ['x86_64', 'aarch64', 'Darwin'] or GetOption('extras'):
- qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'msgq', 'visionipc'])
+ raylib_env = env.Clone()
+
+ raylib_frameworks = []
+ raylib_libs = [common, messaging, visionipc, transformations, 'common', 'msgq', 'visionipc', 'zmq', 'json11', 'pthread', 'raylib']
+ if arch == "Darwin":
+ raylib_frameworks += ['OpenCL', 'CoreVideo', 'Cocoa', 'GLUT', 'CoreFoundation', 'OpenGL', 'IOKit']
+ else:
+ raylib_libs.append('OpenCL')
+
+ raylib_env['LIBPATH'] += [f'#third_party/raylib/{arch}/']
+ raylib_env.Program("watch3", ['watch3.cc', 'raylib/cameraview.cc'], LIBS=raylib_libs, FRAMEWORKS=raylib_frameworks)

selfdrive/ui/raylib/cameraview.cc

@@ -0,0 +1,66 @@
+#include "selfdrive/ui/raylib/cameraview.h"
+
+#include "common/util.h"
+
+const char frame_fragment_shader[] = R"(
+ #version 330 core
+ in vec2 fragTexCoord;
+ uniform sampler2D texture0; // Y plane
+ uniform sampler2D texture1; // UV plane
+ out vec4 fragColor;
+ void main() {
+ float y = texture(texture0, fragTexCoord).r;
+ vec2 uv = texture(texture1, fragTexCoord).ra - 0.5;
+ float r = y + 1.402 * uv.y;
+ float g = y - 0.344 * uv.x - 0.714 * uv.y;
+ float b = y + 1.772 * uv.x;
+ fragColor = vec4(r, g, b, 1.0);
+ }
+)";
+
+CameraView::CameraView(const std::string &name, VisionStreamType type) {
+ client = std::make_unique<VisionIpcClient>(name, type, false);
+ shader = LoadShaderFromMemory(NULL, frame_fragment_shader);
+}
+
+CameraView::~CameraView() {
+ if (textureY.id) UnloadTexture(textureY);
+ if (textureUV.id) UnloadTexture(textureUV);
+ if (shader.id) UnloadShader(shader);
+}
+
+void CameraView::draw(const Rectangle &rec) {
+ if (!ensureConnection()) return;
+
+ auto buffer = client->recv(nullptr, 20);
+ frame = buffer ? buffer : frame;
+ if (!frame) return;
+
+ UpdateTexture(textureY, frame->y);
+ UpdateTexture(textureUV, frame->uv);
+
+ // Calculate scaling factors to maintain aspect ratio
+ float scale = std::min((float)rec.width / frame->width, (float)rec.height / frame->height);
+ float x_offset = rec.x + (rec.width - (frame->width * scale)) / 2;
+ float y_offset = rec.y + (rec.height - (frame->height * scale)) / 2;
+ Rectangle src_rect = {0, 0, (float)frame->width, (float)frame->height};
+ Rectangle dst_rect = {x_offset, y_offset, frame->width * scale, frame->height * scale};
+
+ BeginShaderMode(shader);
+ SetShaderValueTexture(shader, GetShaderLocation(shader, "texture1"), textureUV);
+ DrawTexturePro(textureY, src_rect, dst_rect, Vector2{0, 0}, 0.0, WHITE);
+ EndShaderMode();
+}
+
+bool CameraView::ensureConnection() {
+ if (!client->connected) {
+ frame = nullptr;
+ if (!client->connect(false)) return false;
+
+ // Create textures for Y and UV planes
+ const auto &buf = client->buffers[0];
+ textureY = LoadTextureFromImage(Image{nullptr, (int)buf.stride, (int)buf.height, 1, PIXELFORMAT_UNCOMPRESSED_GRAYSCALE});
+ textureUV = LoadTextureFromImage(Image{nullptr, (int)buf.stride / 2, (int)buf.height / 2, 1, PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA});
+ }
+ return true;
+}

selfdrive/ui/raylib/cameraview.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <memory>
+#include "msgq/visionipc/visionipc_client.h"
+#include "third_party/raylib/include/raylib.h"
+
+class CameraView {
+public:
+ CameraView(const std::string &name, VisionStreamType type);
+ virtual ~CameraView();
+ void draw(const Rectangle &rec);
+
+protected:
+ bool ensureConnection();
+
+ std::unique_ptr<VisionIpcClient> client;
+ Texture2D textureY = {};
+ Texture2D textureUV = {};
+ Shader shader = {};
+ VisionBuf *frame = nullptr;
+};

selfdrive/ui/watch3.cc

@@ -1,33 +1,27 @@
-#include <QApplication>
-#include <QtWidgets>
-
-#include "selfdrive/ui/qt/qt_window.h"
-#include "selfdrive/ui/qt/util.h"
-#include "selfdrive/ui/qt/widgets/cameraview.h"
+#include "selfdrive/ui/raylib/cameraview.h"
 
 int main(int argc, char *argv[]) {
- initApp(argc, argv);
-
- QApplication a(argc, argv);
- QWidget w;
- setMainWindow(&w);
+ SetTraceLogLevel(LOG_NONE);
+ SetConfigFlags(FLAG_WINDOW_RESIZABLE);
+ InitWindow(0, 0, "Watch 3 Cameras");
+ SetTargetFPS(20);
 
- QVBoxLayout *layout = new QVBoxLayout(&w);
- layout->setMargin(0);
- layout->setSpacing(0);
+ CameraView roadCamera("camerad", VISION_STREAM_ROAD);
+ CameraView wideRoadCamera("camerad", VISION_STREAM_WIDE_ROAD);
+ CameraView driverCamera("camerad", VISION_STREAM_DRIVER);
 
- {
- QHBoxLayout *hlayout = new QHBoxLayout();
- layout->addLayout(hlayout);
- hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_ROAD));
- }
+ while (!WindowShouldClose()) {
+ float w = GetScreenWidth(), h = GetScreenHeight();
+ Rectangle roadCameraRec = {0, 0, w, h / 2};
+ Rectangle wideRoadCameraRec = {0, h / 2, w / 2, h / 2};
+  Rectangle driverCameraRec = {w / 2, h / 2, w / 2, h / 2};
 
- {
- QHBoxLayout *hlayout = new QHBoxLayout();
- layout->addLayout(hlayout);
- hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_DRIVER));
- hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_WIDE_ROAD));
+ BeginDrawing();
+ ClearBackground(BLACK);
+ roadCamera.draw(roadCameraRec);
+ wideRoadCamera.draw(wideRoadCameraRec);
+ driverCamera.draw(driverCameraRec);
+ EndDrawing();
  }
-
- return a.exec();
+ return 0;
 }