From d10f7c4b5baec5d49231c2bffeeb2e4f8ecabda3 Mon Sep 17 00:00:00 2001 From: deanlee Date: Mon, 7 Oct 2024 18:55:59 +0800 Subject: [PATCH] raylib cameraview --- selfdrive/ui/SConscript | 14 ++++++- selfdrive/ui/raylib/cameraview.cc | 66 +++++++++++++++++++++++++++++++ selfdrive/ui/raylib/cameraview.h | 21 ++++++++++ selfdrive/ui/watch3.cc | 46 ++++++++++----------- 4 files changed, 119 insertions(+), 28 deletions(-) create mode 100644 selfdrive/ui/raylib/cameraview.cc create mode 100644 selfdrive/ui/raylib/cameraview.h diff --git a/selfdrive/ui/SConscript b/selfdrive/ui/SConscript index 643951fff311cb..26bb22886a9758 100644 --- a/selfdrive/ui/SConscript +++ b/selfdrive/ui/SConscript @@ -1,6 +1,6 @@ import os import json -Import('qt_env', 'arch', 'common', 'messaging', 'visionipc', 'transformations') +Import('env', 'qt_env', 'arch', 'common', 'messaging', 'visionipc', 'transformations') base_libs = [common, messaging, visionipc, transformations, 'm', 'OpenCL', 'ssl', 'crypto', 'pthread'] + qt_env["LIBS"] @@ -109,4 +109,14 @@ if GetOption('extras') and arch != "Darwin": # build watch3 if arch in ['x86_64', 'aarch64', 'Darwin'] or GetOption('extras'): - qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'msgq', 'visionipc']) + raylib_env = env.Clone() + + raylib_frameworks = [] + raylib_libs = [common, messaging, visionipc, transformations, 'common', 'msgq', 'visionipc', 'zmq', 'json11', 'pthread', 'raylib'] + if arch == "Darwin": + raylib_frameworks += ['OpenCL', 'CoreVideo', 'Cocoa', 'GLUT', 'CoreFoundation', 'OpenGL', 'IOKit'] + else: + raylib_libs.append('OpenCL') + + raylib_env['LIBPATH'] += [f'#third_party/raylib/{arch}/'] + raylib_env.Program("watch3", ['watch3.cc', 'raylib/cameraview.cc'], LIBS=raylib_libs, FRAMEWORKS=raylib_frameworks) diff --git a/selfdrive/ui/raylib/cameraview.cc b/selfdrive/ui/raylib/cameraview.cc new file mode 100644 index 00000000000000..0d0955eb1ce5b0 --- /dev/null +++ b/selfdrive/ui/raylib/cameraview.cc @@ -0,0 +1,66 @@ +#include "selfdrive/ui/raylib/cameraview.h" + +#include "common/util.h" + +const char frame_fragment_shader[] = R"( + #version 330 core + in vec2 fragTexCoord; + uniform sampler2D texture0; // Y plane + uniform sampler2D texture1; // UV plane + out vec4 fragColor; + void main() { + float y = texture(texture0, fragTexCoord).r; + vec2 uv = texture(texture1, fragTexCoord).ra - 0.5; + float r = y + 1.402 * uv.y; + float g = y - 0.344 * uv.x - 0.714 * uv.y; + float b = y + 1.772 * uv.x; + fragColor = vec4(r, g, b, 1.0); + } +)"; + +CameraView::CameraView(const std::string &name, VisionStreamType type) { + client = std::make_unique(name, type, false); + shader = LoadShaderFromMemory(NULL, frame_fragment_shader); +} + +CameraView::~CameraView() { + if (textureY.id) UnloadTexture(textureY); + if (textureUV.id) UnloadTexture(textureUV); + if (shader.id) UnloadShader(shader); +} + +void CameraView::draw(const Rectangle &rec) { + if (!ensureConnection()) return; + + auto buffer = client->recv(nullptr, 20); + frame = buffer ? buffer : frame; + if (!frame) return; + + UpdateTexture(textureY, frame->y); + UpdateTexture(textureUV, frame->uv); + + // Calculate scaling factors to maintain aspect ratio + float scale = std::min((float)rec.width / frame->width, (float)rec.height / frame->height); + float x_offset = rec.x + (rec.width - (frame->width * scale)) / 2; + float y_offset = rec.y + (rec.height - (frame->height * scale)) / 2; + Rectangle src_rect = {0, 0, (float)frame->width, (float)frame->height}; + Rectangle dst_rect = {x_offset, y_offset, frame->width * scale, frame->height * scale}; + + BeginShaderMode(shader); + SetShaderValueTexture(shader, GetShaderLocation(shader, "texture1"), textureUV); + DrawTexturePro(textureY, src_rect, dst_rect, Vector2{0, 0}, 0.0, WHITE); + EndShaderMode(); +} + +bool CameraView::ensureConnection() { + if (!client->connected) { + frame = nullptr; + if (!client->connect(false)) return false; + + // Create textures for Y and UV planes + const auto &buf = client->buffers[0]; + textureY = LoadTextureFromImage(Image{nullptr, (int)buf.stride, (int)buf.height, 1, PIXELFORMAT_UNCOMPRESSED_GRAYSCALE}); + textureUV = LoadTextureFromImage(Image{nullptr, (int)buf.stride / 2, (int)buf.height / 2, 1, PIXELFORMAT_UNCOMPRESSED_GRAY_ALPHA}); + } + return true; +} diff --git a/selfdrive/ui/raylib/cameraview.h b/selfdrive/ui/raylib/cameraview.h new file mode 100644 index 00000000000000..4ff33c61a28fb0 --- /dev/null +++ b/selfdrive/ui/raylib/cameraview.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include "msgq/visionipc/visionipc_client.h" +#include "third_party/raylib/include/raylib.h" + +class CameraView { +public: + CameraView(const std::string &name, VisionStreamType type); + virtual ~CameraView(); + void draw(const Rectangle &rec); + +protected: + bool ensureConnection(); + + std::unique_ptr client; + Texture2D textureY = {}; + Texture2D textureUV = {}; + Shader shader = {}; + VisionBuf *frame = nullptr; +}; diff --git a/selfdrive/ui/watch3.cc b/selfdrive/ui/watch3.cc index 258e2a7bd6ff9c..2e17c013fa1ee1 100644 --- a/selfdrive/ui/watch3.cc +++ b/selfdrive/ui/watch3.cc @@ -1,33 +1,27 @@ -#include -#include - -#include "selfdrive/ui/qt/qt_window.h" -#include "selfdrive/ui/qt/util.h" -#include "selfdrive/ui/qt/widgets/cameraview.h" +#include "selfdrive/ui/raylib/cameraview.h" int main(int argc, char *argv[]) { - initApp(argc, argv); - - QApplication a(argc, argv); - QWidget w; - setMainWindow(&w); + SetTraceLogLevel(LOG_NONE); + SetConfigFlags(FLAG_WINDOW_RESIZABLE); + InitWindow(0, 0, "Watch 3 Cameras"); + SetTargetFPS(20); - QVBoxLayout *layout = new QVBoxLayout(&w); - layout->setMargin(0); - layout->setSpacing(0); + CameraView roadCamera("camerad", VISION_STREAM_ROAD); + CameraView wideRoadCamera("camerad", VISION_STREAM_WIDE_ROAD); + CameraView driverCamera("camerad", VISION_STREAM_DRIVER); - { - QHBoxLayout *hlayout = new QHBoxLayout(); - layout->addLayout(hlayout); - hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_ROAD)); - } + while (!WindowShouldClose()) { + float w = GetScreenWidth(), h = GetScreenHeight(); + Rectangle roadCameraRec = {0, 0, w, h / 2}; + Rectangle wideRoadCameraRec = {0, h / 2, w / 2, h / 2}; + Rectangle driverCameraRec = {w / 2, h / 2, w / 2, h / 2}; - { - QHBoxLayout *hlayout = new QHBoxLayout(); - layout->addLayout(hlayout); - hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_DRIVER)); - hlayout->addWidget(new CameraWidget("camerad", VISION_STREAM_WIDE_ROAD)); + BeginDrawing(); + ClearBackground(BLACK); + roadCamera.draw(roadCameraRec); + wideRoadCamera.draw(wideRoadCameraRec); + driverCamera.draw(driverCameraRec); + EndDrawing(); } - - return a.exec(); + return 0; }