diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bf95d67..a17ce4b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,7 +59,7 @@ importers: version: 20.11.20 '@typescript-eslint/eslint-plugin': specifier: ^7.1.0 - version: 7.1.0(@typescript-eslint/parser@7.1.0)(eslint@8.57.0)(typescript@5.3.3) + version: 7.1.0(@typescript-eslint/parser@7.1.0(eslint@8.57.0)(typescript@5.3.3))(eslint@8.57.0)(typescript@5.3.3) '@typescript-eslint/parser': specifier: ^7.1.0 version: 7.1.0(eslint@8.57.0)(typescript@5.3.3) @@ -92,7 +92,7 @@ importers: version: 3.2.5 solid-devtools: specifier: ^0.29.3 - version: 0.29.3(solid-js@1.8.15)(vite@5.2.7) + version: 0.29.3(solid-js@1.8.15)(vite@5.2.7(@types/node@20.11.20)) tailwindcss: specifier: ^3.4.1 version: 3.4.1 @@ -104,7 +104,7 @@ importers: version: 5.2.7(@types/node@20.11.20) vite-plugin-solid: specifier: ^2.10.1 - version: 2.10.1(solid-js@1.8.15)(vite@5.2.7) + version: 2.10.1(solid-js@1.8.15)(vite@5.2.7(@types/node@20.11.20)) packages: @@ -2355,7 +2355,6 @@ packages: tauri-plugin-websocket-api@https://codeload.github.com/tauri-apps/tauri-plugin-websocket/tar.gz/c7ac2dfb949db67397931878aeda034e3921c514: resolution: {tarball: https://codeload.github.com/tauri-apps/tauri-plugin-websocket/tar.gz/c7ac2dfb949db67397931878aeda034e3921c514} - name: tauri-plugin-websocket-api version: 0.0.0 text-table@0.2.0: @@ -3128,7 +3127,7 @@ snapshots: '@types/semver@7.5.8': {} - '@typescript-eslint/eslint-plugin@7.1.0(@typescript-eslint/parser@7.1.0)(eslint@8.57.0)(typescript@5.3.3)': + '@typescript-eslint/eslint-plugin@7.1.0(@typescript-eslint/parser@7.1.0(eslint@8.57.0)(typescript@5.3.3))(eslint@8.57.0)(typescript@5.3.3)': dependencies: '@eslint-community/regexpp': 4.10.0 '@typescript-eslint/parser': 7.1.0(eslint@8.57.0)(typescript@5.3.3) @@ -3143,6 +3142,7 @@ snapshots: natural-compare: 1.4.0 semver: 7.6.0 ts-api-utils: 1.2.1(typescript@5.3.3) + optionalDependencies: typescript: 5.3.3 transitivePeerDependencies: - supports-color @@ -3155,6 +3155,7 @@ snapshots: '@typescript-eslint/visitor-keys': 7.1.0 debug: 4.3.4 eslint: 8.57.0 + optionalDependencies: typescript: 5.3.3 transitivePeerDependencies: - supports-color @@ -3176,6 +3177,7 @@ snapshots: debug: 4.3.4 eslint: 8.57.0 ts-api-utils: 1.2.1(typescript@5.3.3) + optionalDependencies: typescript: 5.3.3 transitivePeerDependencies: - supports-color @@ -3194,6 +3196,7 @@ snapshots: minimatch: 9.0.3 semver: 7.6.0 ts-api-utils: 1.2.1(typescript@5.3.3) + optionalDependencies: typescript: 5.3.3 transitivePeerDependencies: - supports-color @@ -3208,6 +3211,7 @@ snapshots: minimatch: 9.0.3 semver: 7.6.0 ts-api-utils: 1.2.1(typescript@5.3.3) + optionalDependencies: typescript: 5.3.3 transitivePeerDependencies: - supports-color @@ -4491,8 +4495,9 @@ snapshots: postcss-load-config@4.0.2(postcss@8.4.35): dependencies: lilconfig: 3.1.1 - postcss: 8.4.35 yaml: 2.4.0 + optionalDependencies: + postcss: 8.4.35 postcss-merge-longhand@6.0.5(postcss@8.4.35): dependencies: @@ -4788,7 +4793,7 @@ snapshots: slash@3.0.0: {} - solid-devtools@0.29.3(solid-js@1.8.15)(vite@5.2.7): + solid-devtools@0.29.3(solid-js@1.8.15)(vite@5.2.7(@types/node@20.11.20)): dependencies: '@babel/core': 7.23.9 '@babel/plugin-syntax-typescript': 7.23.3(@babel/core@7.23.9) @@ -4796,6 +4801,7 @@ snapshots: '@solid-devtools/debugger': 0.23.3(solid-js@1.8.15) '@solid-devtools/shared': 0.13.1(solid-js@1.8.15) solid-js: 1.8.15 + optionalDependencies: vite: 5.2.7(@types/node@20.11.20) transitivePeerDependencies: - supports-color @@ -5051,7 +5057,7 @@ snapshots: validate-html-nesting@1.2.2: {} - vite-plugin-solid@2.10.1(solid-js@1.8.15)(vite@5.2.7): + vite-plugin-solid@2.10.1(solid-js@1.8.15)(vite@5.2.7(@types/node@20.11.20)): dependencies: '@babel/core': 7.23.9 '@types/babel__core': 7.20.5 @@ -5060,21 +5066,21 @@ snapshots: solid-js: 1.8.15 solid-refresh: 0.6.3(solid-js@1.8.15) vite: 5.2.7(@types/node@20.11.20) - vitefu: 0.2.5(vite@5.2.7) + vitefu: 0.2.5(vite@5.2.7(@types/node@20.11.20)) transitivePeerDependencies: - supports-color vite@5.2.7(@types/node@20.11.20): dependencies: - '@types/node': 20.11.20 esbuild: 0.20.2 postcss: 8.4.38 rollup: 4.13.2 optionalDependencies: + '@types/node': 20.11.20 fsevents: 2.3.3 - vitefu@0.2.5(vite@5.2.7): - dependencies: + vitefu@0.2.5(vite@5.2.7(@types/node@20.11.20)): + optionalDependencies: vite: 5.2.7(@types/node@20.11.20) which-boxed-primitive@1.0.2: diff --git a/qmod/assets.cmake b/qmod/assets.cmake index 23ad4d0..7f24770 100644 --- a/qmod/assets.cmake +++ b/qmod/assets.cmake @@ -40,7 +40,7 @@ if(EXISTS ${ASSETS_DIRECTORY}) add_custom_command( OUTPUT ${PREPENDED_ASSETS_DIR}/${ASSET} - COMMAND ${CMAKE_COMMAND} -E cat ${ASSETS_DIRECTORY}/${ASSET} >> ${PREPENDED_ASSETS_DIR}/${ASSET} + COMMAND ${CMAKE_COMMAND} -E cat ${ASSETS_DIRECTORY}/${ASSET} > ${PREPENDED_ASSETS_DIR}/${ASSET} COMMAND ${CMAKE_COMMAND} -E echo_append " " >> ${PREPENDED_ASSETS_DIR}/${ASSET} DEPENDS ${ASSETS_DIRECTORY}/${ASSET} ) diff --git a/qmod/include/CameraController.hpp b/qmod/include/CameraController.hpp index f696481..403250f 100644 --- a/qmod/include/CameraController.hpp +++ b/qmod/include/CameraController.hpp @@ -1,18 +1,18 @@ #pragma once -#ifdef BEAT_SABER +#include "UnityEngine/Camera.hpp" +#include "UnityEngine/GameObject.hpp" +#include "UnityEngine/MonoBehaviour.hpp" +#include "UnityEngine/Transform.hpp" +#include "UnityEngine/Vector2.hpp" #include "custom-types/shared/macros.hpp" - #include "sombrero/shared/FastVector2.hpp" #include "sombrero/shared/FastVector3.hpp" -#include "UnityEngine/MonoBehaviour.hpp" -#include "UnityEngine/Vector2.hpp" -#include "UnityEngine/Transform.hpp" -#include "UnityEngine/GameObject.hpp" -#include "UnityEngine/Camera.hpp" +#ifdef BEAT_SABER #include "GlobalNamespace/VRController.hpp" #include "HMUI/UIKeyboard.hpp" +#endif DECLARE_CLASS_CODEGEN(QRUE, CameraController, UnityEngine::MonoBehaviour, DECLARE_DEFAULT_CTOR(); @@ -23,19 +23,15 @@ DECLARE_CLASS_CODEGEN(QRUE, CameraController, UnityEngine::MonoBehaviour, DECLARE_INSTANCE_METHOD(void, Rotate, Sombrero::FastVector2); DECLARE_INSTANCE_METHOD(void, Move, Sombrero::FastVector3); -#ifdef UNITY_2021 - DECLARE_INSTANCE_FIELD(UnityEngine::Camera*, customCamera); - DECLARE_INSTANCE_FIELD_DEFAULT(bool, recording, false); -#endif DECLARE_INSTANCE_FIELD(float, lastTime); DECLARE_INSTANCE_FIELD(float, lastMovement); DECLARE_INSTANCE_FIELD(bool, backspaceHold); DECLARE_INSTANCE_FIELD(float, backspaceHoldStart); DECLARE_INSTANCE_FIELD(UnityEngine::Vector2, lastPos); +#ifdef BEAT_SABER DECLARE_INSTANCE_FIELD(GlobalNamespace::VRController*, controller0); DECLARE_INSTANCE_FIELD(GlobalNamespace::VRController*, controller1); - DECLARE_INSTANCE_FIELD(UnityEngine::Transform*, parentTransform); - DECLARE_INSTANCE_FIELD(UnityEngine::Transform*, childTransform); +#endif ) #ifdef UNITY_2021 @@ -59,6 +55,7 @@ DECLARE_CLASS_CODEGEN(QRUE, CameraStreamer, UnityEngine::MonoBehaviour, DECLARE_INSTANCE_FIELD_DEFAULT(int, height, -1); DECLARE_INSTANCE_FIELD_DEFAULT(bool, initializedEncoder, false); DECLARE_INSTANCE_FIELD_DEFAULT(bool, initializedEgl, false); + DECLARE_INSTANCE_FIELD_DEFAULT(bool, stopping, false); DECLARE_INSTANCE_FIELD_DEFAULT(int, framesProcessing, 0); public: @@ -72,10 +69,13 @@ DECLARE_CLASS_CODEGEN(QRUE, CameraStreamer, UnityEngine::MonoBehaviour, private: AMediaCodec* encoder; - bool stopThread = false; + bool threadRunning = false; std::thread threadInst; void EncodingThread(); + bool shouldRestart = false; + int restartWidth, restartHeight, restartBitrate; + static inline int maxId = 0; static inline std::shared_mutex idMapMutex; static inline std::unordered_map idMap{}; @@ -83,20 +83,17 @@ DECLARE_CLASS_CODEGEN(QRUE, CameraStreamer, UnityEngine::MonoBehaviour, #endif extern bool fpfcEnabled; -#ifdef UNITY_2021 -extern Sombrero::FastVector3 fpfcPos; -extern Sombrero::FastVector3 fpfcRot; -#endif - -extern bool click; -extern bool clickOnce; -extern HMUI::UIKeyboard* keyboardOpen; extern float rotateSensitivity; extern float moveSensitivity; extern float clickTime; extern float movementThreshold; -UnityEngine::GameObject* GetHovered(); +#ifdef BEAT_SABER +extern bool click; +extern bool clickOnce; +extern HMUI::UIKeyboard* keyboardOpen; +UnityEngine::GameObject* GetHovered(); #endif + diff --git a/qmod/src/CameraController.cpp b/qmod/src/CameraController.cpp index cd0e263..73d9428 100644 --- a/qmod/src/CameraController.cpp +++ b/qmod/src/CameraController.cpp @@ -1,7 +1,18 @@ -#ifdef BEAT_SABER #include "CameraController.hpp" +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "assets.hpp" #include "main.hpp" +#include "media/NdkMediaFormat.h" #include "sombrero/shared/FastVector2.hpp" #include "sombrero/shared/FastVector3.hpp" @@ -10,15 +21,10 @@ DEFINE_TYPE(QRUE, CameraController); DEFINE_TYPE(QRUE, CameraStreamer); #endif -bool fpfcEnabled = true; -#ifdef UNITY_2021 -Sombrero::FastVector3 fpfcPos; -Sombrero::FastVector3 fpfcRot; -#endif +using namespace QRUE; +using namespace UnityEngine; -bool click = false; -bool clickOnce = false; -HMUI::UIKeyboard* keyboardOpen = nullptr; +bool fpfcEnabled = true; float rotateSensitivity = 1; float moveSensitivity = 1; @@ -26,12 +32,26 @@ float clickTime = 0.2; float movementThreshold = 1; float backspaceTime = 0.5; +#ifdef BEAT_SABER +bool click = false; +bool clickOnce = false; +HMUI::UIKeyboard* keyboardOpen = nullptr; + +#include "GlobalNamespace/FirstPersonFlyingController.hpp" +#include "GlobalNamespace/PauseController.hpp" +#include "GlobalNamespace/PauseMenuManager.hpp" +#include "GlobalNamespace/UIKeyboardManager.hpp" +#include "GlobalNamespace/VRCenterAdjust.hpp" #include "UnityEngine/EventSystems/PointerEventData.hpp" #include "VRUIControls/VRInputModule.hpp" +#include "VRUIControls/VRLaserPointer.hpp" +#include "VRUIControls/VRPointer.hpp" + +using namespace GlobalNamespace; SafePtrUnity latestInputModule; -UnityEngine::GameObject* GetHovered() { +GameObject* GetHovered() { if (!latestInputModule) return nullptr; auto eventData = latestInputModule->GetLastPointerEventData(-1); @@ -40,63 +60,35 @@ UnityEngine::GameObject* GetHovered() { return nullptr; } -UnityEngine::GameObject* GetPauseMenu() { - if (auto ret = UnityEngine::GameObject::Find("PauseMenu")) +GameObject* GetPauseMenu() { + if (auto ret = GameObject::Find("PauseMenu")) return ret; - else if (auto ret = UnityEngine::GameObject::Find("TutorialPauseMenu")) + else if (auto ret = GameObject::Find("TutorialPauseMenu")) return ret; - else if (auto ret = UnityEngine::GameObject::Find("MultiplayerLocalActivePlayerInGameMenuViewController")) + else if (auto ret = GameObject::Find("MultiplayerLocalActivePlayerInGameMenuViewController")) return ret; else return nullptr; } - -using namespace QRUE; -using namespace UnityEngine; -using namespace GlobalNamespace; - -#include "GlobalNamespace/FirstPersonFlyingController.hpp" -#include "GlobalNamespace/VRCenterAdjust.hpp" -#include "VRUIControls/VRLaserPointer.hpp" -#include "VRUIControls/VRPointer.hpp" +#endif void CameraController::OnEnable() { fpfcEnabled = true; LOG_INFO("CameraController enable"); - childTransform = get_transform(); -#ifdef UNITY_2021 - fpfcPos = {0, 1.5, 0}; - fpfcRot = {}; -#else - parentTransform = childTransform->GetParent(); - - if (!parentTransform) { - set_enabled(false); - return; - } - - static auto disablePositionalTracking = il2cpp_utils::resolve_icall("UnityEngine.XR.InputTracking::set_disablePositionalTracking"); - disablePositionalTracking(true); - - parentTransform->set_position({0, 1.5, 0}); - parentTransform->set_eulerAngles({0, 0, 0}); - childTransform->set_localPosition({0, 0, 0}); - childTransform->set_localEulerAngles({0, 0, 0}); -#endif - +#ifdef BEAT_SABER #ifdef UNITY_2021 // in level if (auto pauseMenu = GetPauseMenu()) { // can't just search for "MenuControllers" because there are two, we need // the one that's a child of the pause menu - auto transform = pauseMenu->get_transform()->Find("MenuControllers"); + auto transform = pauseMenu->transform->Find("MenuControllers"); controller0 = transform->Find("ControllerLeft")->GetComponent(); controller1 = transform->Find("ControllerRight")->GetComponent(); // in main menu } else if (auto objectsSource = Object::FindObjectOfType()) { objectsSource->_centerAdjust->ResetRoom(); - objectsSource->_centerAdjust->set_enabled(false); + objectsSource->_centerAdjust->enabled = false; for (auto gameObject : objectsSource->_controllerModels) { if (gameObject) gameObject->SetActive(false); @@ -146,24 +138,14 @@ void CameraController::OnEnable() { if (auto pointer = controller1->get_transform()->Find("MenuHandle")) pointer->get_gameObject()->SetActive(false); } +#endif } void CameraController::OnDisable() { fpfcEnabled = false; LOG_INFO("CameraController disable"); -#ifndef UNITY_2021 - if (!parentTransform) - return; - - // reverse of enable - static auto disablePositionalTracking = il2cpp_utils::resolve_icall("UnityEngine.XR.InputTracking::set_disablePositionalTracking"); - disablePositionalTracking(false); - - parentTransform->set_position({0, 0, 0}); - parentTransform->set_eulerAngles({0, 0, 0}); -#endif - +#ifdef BEAT_SABER #ifdef UNITY_2021 if (auto pauseMenu = GetPauseMenu()) { LOG_DEBUG("Using controllers from pause menu"); @@ -221,38 +203,15 @@ void CameraController::OnDisable() { pointer->get_gameObject()->SetActive(true); } else LOG_INFO("Failed to find menu controllers for FPFC"); +#endif } - -#include "GlobalNamespace/DepthTextureController.hpp" -#include "GlobalNamespace/MainCamera.hpp" -#include "GlobalNamespace/MainCameraCullingMask.hpp" -#include "GlobalNamespace/PauseController.hpp" -#include "GlobalNamespace/PauseMenuManager.hpp" -#include "GlobalNamespace/UIKeyboardManager.hpp" #include "System/Action.hpp" #include "System/Action_1.hpp" -#include "UnityEngine/Camera.hpp" -#include "UnityEngine/CameraType.hpp" -#include "UnityEngine/FilterMode.hpp" #include "UnityEngine/Input.hpp" #include "UnityEngine/KeyCode.hpp" -#include "UnityEngine/RenderTexture.hpp" -#include "UnityEngine/RenderTextureFormat.hpp" -#include "UnityEngine/RenderTextureReadWrite.hpp" -#include "UnityEngine/StereoTargetEyeMask.hpp" -#include "UnityEngine/TextureWrapMode.hpp" #include "UnityEngine/Time.hpp" #include "UnityEngine/Touch.hpp" -RenderTexture* CreateCameraTexture(int w, int h) { - auto texture = RenderTexture::New_ctor(w, h, 24, RenderTextureFormat::Default, RenderTextureReadWrite::Default); - Object::DontDestroyOnLoad(texture); - texture->wrapMode = TextureWrapMode::Clamp; - texture->filterMode = FilterMode::Bilinear; - texture->Create(); - return texture; -} - void CameraController::Update() { if (Input::GetKeyDown(KeyCode::X)) { LOG_INFO("Disabling FPFC due to X press (reenable with Z)"); @@ -262,65 +221,8 @@ void CameraController::Update() { return; } -#ifdef UNITY_2021 - if (Input::GetKeyDown(KeyCode::V)) { - if (recording) { - LOG_INFO("Disabling recording"); - - Object::DestroyImmediate(customCamera->gameObject); - customCamera = nullptr; - recording = false; - } else { - LOG_INFO("Enabling recording"); - auto mainCamera = GetComponent(); - if (auto comp = mainCamera->GetComponent()) - Object::DestroyImmediate(comp); - customCamera = Object::Instantiate(mainCamera); - customCamera->gameObject->active = false; - customCamera->enabled = true; - - while (customCamera->transform->childCount > 0) - Object::DestroyImmediate(customCamera->transform->GetChild(0)->gameObject); - - if (auto comp = customCamera->GetComponent()) - Object::DestroyImmediate(comp); - if (auto comp = customCamera->GetComponent()) - Object::DestroyImmediate(comp); - if (auto comp = customCamera->GetComponent()) - Object::DestroyImmediate(comp); - if (auto comp = customCamera->GetComponent("CameraController")) - Object::DestroyImmediate(comp); - - customCamera->clearFlags = mainCamera->clearFlags; - customCamera->nearClipPlane = mainCamera->nearClipPlane; - customCamera->farClipPlane = mainCamera->farClipPlane; - customCamera->backgroundColor = mainCamera->backgroundColor; - customCamera->hideFlags = mainCamera->hideFlags; - customCamera->depthTextureMode = mainCamera->depthTextureMode; - customCamera->cullingMask = mainCamera->cullingMask; - // Makes the camera render after the main - customCamera->depth = mainCamera->depth + 1; - - customCamera->stereoTargetEye = StereoTargetEyeMask::None; - - int w = 1080; - int h = 720; - auto tex = CreateCameraTexture(w, h); - customCamera->targetTexture = tex; - auto streamer = customCamera->gameObject->AddComponent(); - streamer->texture = (uintptr_t) tex->GetNativeTexturePtr().m_value.convert(); - streamer->onOutputFrame = [](uint8_t* data, size_t length) { - // LOG_DEBUG("output frame callback!"); - }; - streamer->Init(w, h, 1000000); - customCamera->gameObject->active = true; - - recording = true; - } - } -#endif - float time = UnityEngine::Time::get_time(); + auto transform = get_transform(); if (Input::get_touchCount() > 0) { auto touch = Input::GetTouch(0); @@ -352,17 +254,17 @@ void CameraController::Update() { if (!keyboardOpen) { Sombrero::FastVector3 movement = {0}; if (Input::GetKey(KeyCode::W)) - movement = movement + childTransform->get_forward(); + movement = movement + transform->get_forward(); if (Input::GetKey(KeyCode::S)) - movement = movement - childTransform->get_forward(); + movement = movement - transform->get_forward(); if (Input::GetKey(KeyCode::D)) - movement = movement + childTransform->get_right(); + movement = movement + transform->get_right(); if (Input::GetKey(KeyCode::A)) - movement = movement - childTransform->get_right(); + movement = movement - transform->get_right(); if (Input::GetKey(KeyCode::Space)) - movement = movement + childTransform->get_up(); + movement = movement + transform->get_up(); if (Input::GetKey(KeyCode::LeftControl) || Input::GetKey(KeyCode::RightControl)) - movement = movement - childTransform->get_up(); + movement = movement - transform->get_up(); if (movement != Sombrero::FastVector3::zero()) Move(movement); @@ -407,46 +309,24 @@ void CameraController::Update() { } if (controller0 && controller1) { - controller0->get_transform()->SetPositionAndRotation(childTransform->get_position(), childTransform->get_rotation()); - controller1->get_transform()->SetPositionAndRotation(childTransform->get_position(), childTransform->get_rotation()); + controller0->get_transform()->SetPositionAndRotation(transform->get_position(), transform->get_rotation()); + controller1->get_transform()->SetPositionAndRotation(transform->get_position(), transform->get_rotation()); } } void CameraController::Rotate(Sombrero::FastVector2 delta) { delta = delta * rotateSensitivity * 20; lastMovement += delta.get_magnitude(); -#ifdef UNITY_2021 - fpfcRot += Sombrero::FastVector3{-delta.y, delta.x, 0}; -#else - Sombrero::FastVector3 prev = parentTransform->get_eulerAngles(); - parentTransform->set_eulerAngles(prev + Sombrero::FastVector3{-delta.y, delta.x, 0}); -#endif + Sombrero::FastVector3 prev = get_transform()->get_eulerAngles(); + get_transform()->set_eulerAngles(prev + Sombrero::FastVector3{-delta.y, delta.x, 0}); } void CameraController::Move(Sombrero::FastVector3 delta) { delta = delta * moveSensitivity / 50; -#ifdef UNITY_2021 - fpfcPos += delta; -#else - Sombrero::FastVector3 prev = parentTransform->get_position(); - parentTransform->set_position(prev + delta); -#endif + Sombrero::FastVector3 prev = get_transform()->get_position(); + get_transform()->set_position(prev + delta); } -// should be easily ported to 2019 as well -#ifdef UNITY_2021 -#include -#include -#include -#include -#include -#include - -#include "assets.hpp" -#include "media/NdkMediaFormat.h" - -static FILE* f; - // clang-format off // if you change these, it will probably break EGLint const attribs[] = { @@ -735,6 +615,11 @@ namespace RenderThread { LOG_INFO("Initializing EGL for streamer {}", id); auto streamer = CameraStreamer::GetById(id); + if (!streamer->initializedEgl || streamer->stopping) { + EGLState::Load(); + return; + } + streamer->surface = eglCreateWindowSurface(display, config, streamer->window, 0); EGL_BOOL_CHECK(eglMakeCurrent(display, streamer->surface, streamer->surface, context), "make current"); @@ -755,6 +640,11 @@ namespace RenderThread { EGLState::Save(); auto streamer = CameraStreamer::GetById(id); + if (!streamer->initializedEgl || streamer->stopping) { + EGLState::Load(); + return; + } + EGL_BOOL_CHECK(eglMakeCurrent(display, streamer->surface, streamer->surface, context), "make current"); #ifdef NO_GAMMA_SHADER @@ -794,21 +684,24 @@ namespace RenderThread { EGLState::Save(); auto streamer = CameraStreamer::GetById(id); - eglDestroySurface(display, streamer->surface); + if (streamer->surface) + eglDestroySurface(display, streamer->surface); streamer->surface = nullptr; #ifdef NO_GAMMA_SHADER GLuint fboId = streamer->buffer; - glDeleteFramebuffers(1, &fboId); + if (fboId >= 0) + glDeleteFramebuffers(1, &fboId); #else GLuint vboId = streamer->buffer; - glDeleteVertexArrays(1, &vboId); + if (vboId >= 0) + glDeleteVertexArrays(1, &vboId); #endif streamer->buffer = -1; - streamer->FinishStop(); - EGLState::Load(); + + streamer->FinishStop(); } } @@ -827,7 +720,6 @@ void IssuePluginEvent(void (*function)(int), int id) { // uint8_t* data = AMediaCodec_getOutputBuffer(codec, index, &outSize); // streamer->framesProcessing--; // streamer->onOutputFrame(data, bufferInfo->size); -// fwrite(data, sizeof(uint8_t), bufferInfo->size, f); // AMediaCodec_releaseOutputBuffer(codec, index, false); // } @@ -846,13 +738,21 @@ void IssuePluginEvent(void (*function)(int), int id) { // } void CameraStreamer::Init(int width, int height, int bitrate) { + // this threading stuff is probably still unsafe because threads are terrible + if (stopping) { + LOG_DEBUG("Streamer is stopping, should restart"); + shouldRestart = true; + restartWidth = width; + restartHeight = height; + restartBitrate = bitrate; + return; + } + shouldRestart = false; + if (initializedEncoder) return; std::unique_lock lock(idMapMutex); - if (id >= 0) - idMap.erase(id); - id = maxId++; idMap[id] = this; lock.unlock(); @@ -916,31 +816,42 @@ void CameraStreamer::Init(int width, int height, int bitrate) { return; } - f = fopen("/sdcard/test.h264", "wb"); - if (!f) - logger.error("Could not open /sdcard/test.h264"); - framesProcessing = 0; initializedEncoder = true; - stopThread = false; + stopping = false; threadInst = std::thread(&CameraStreamer::EncodingThread, this); LOG_INFO("Finished initializing encoder"); } void CameraStreamer::Stop() { + if (stopping) + return; + LOG_INFO("Stopping streamer {}", id); + stopping = true; + + bool waitForPlugin = initializedEgl; if (initializedEgl) IssuePluginEvent(RenderThread::StopId, id); - initializedEgl = false; - stopThread = true; - threadInst.join(); - // wait until the event happens before we do the rest + + if (threadInst.joinable()) + threadInst.join(); + + // if called, wait until the event happens before we do the rest + if (!waitForPlugin) + FinishStop(); } void CameraStreamer::FinishStop() { + initializedEgl = false; + + // make sure encoding thread has stopped + while (threadRunning) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + if (initializedEncoder) { ANativeWindow_release(window); AMediaCodec_stop(encoder); @@ -949,18 +860,19 @@ void CameraStreamer::FinishStop() { initializedEncoder = false; } - std::unique_lock lock(idMapMutex); - LOG_INFO("Finished stopping streamer {}", id); + LOG_INFO("Finished stopping streamer {} {}", id, shouldRestart); + std::unique_lock lock(idMapMutex); idMap.erase(id); id = -1; - - fclose(f); - f = nullptr; + stopping = false; } void CameraStreamer::Update() { - if (!initializedEncoder) + if (!stopping && shouldRestart) + Init(restartWidth, restartHeight, restartBitrate); + + if (!initializedEncoder || stopping) return; if (!initializedEgl) { @@ -984,10 +896,11 @@ CameraStreamer* CameraStreamer::GetById(int id) { void CameraStreamer::EncodingThread() { AMediaCodecBufferInfo bufferInfo; + threadRunning = true; LOG_DEBUG("starting thread"); - while (!stopThread) { + while (!stopping) { ssize_t index = AMediaCodec_dequeueOutputBuffer(encoder, &bufferInfo, 10000); if (index >= 0) { @@ -996,6 +909,8 @@ void CameraStreamer::EncodingThread() { uint8_t* data = AMediaCodec_getOutputBuffer(encoder, index, &outSize); // note: make sure to write the CODEC_CONFIG to have a valid raw .h264 file + if (bufferInfo.flags & AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG) + LOG_DEBUG("codec_config packet {}", bufferInfo.size); // not sure what outSize represents, but it seems wrong if (bufferInfo.size != 0) { @@ -1003,18 +918,15 @@ void CameraStreamer::EncodingThread() { // bufferInfo.offset seems to always be 0 for encoding onOutputFrame(data, bufferInfo.size); - fwrite(data, sizeof(uint8_t), bufferInfo.size, f); } AMediaCodec_releaseOutputBuffer(encoder, index, false); - if ((bufferInfo.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) != 0) + if (bufferInfo.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) break; } } - LOG_DEBUG("stopping thread"); + threadRunning = false; + LOG_DEBUG("stopped thread"); } -#endif - -#endif diff --git a/qmod/src/MainThreadRunner.cpp b/qmod/src/MainThreadRunner.cpp index bc95158..5181af3 100644 --- a/qmod/src/MainThreadRunner.cpp +++ b/qmod/src/MainThreadRunner.cpp @@ -40,14 +40,6 @@ MainThreadRunner* getUnityHandle() { } void MainThreadRunner::Update() { -#ifdef BEAT_SABER - // listen for fpfc enable key (C) - if (UnityEngine::Input::GetKey(UnityEngine::KeyCode::Z)) { - fpfcEnabled = true; - EnableFPFC(); - } -#endif - if (scheduledFunctions.empty()) return; diff --git a/qmod/src/main.cpp b/qmod/src/main.cpp index a433441..3120721 100644 --- a/qmod/src/main.cpp +++ b/qmod/src/main.cpp @@ -1,12 +1,24 @@ #include "main.hpp" +#include +#include + #include "CameraController.hpp" #include "MainThreadRunner.hpp" +#include "UnityEngine/Color.hpp" #include "UnityEngine/Events/UnityAction_2.hpp" +#include "UnityEngine/FilterMode.hpp" #include "UnityEngine/GameObject.hpp" +#include "UnityEngine/Matrix4x4.hpp" +#include "UnityEngine/Rect.hpp" +#include "UnityEngine/RenderTexture.hpp" +#include "UnityEngine/RenderTextureFormat.hpp" +#include "UnityEngine/RenderTextureReadWrite.hpp" #include "UnityEngine/SceneManagement/LoadSceneMode.hpp" #include "UnityEngine/SceneManagement/Scene.hpp" #include "UnityEngine/SceneManagement/SceneManager.hpp" +#include "UnityEngine/StereoTargetEyeMask.hpp" +#include "UnityEngine/TextureWrapMode.hpp" #include "beatsaber-hook/shared/config/config-utils.hpp" #include "beatsaber-hook/shared/utils/hooking.hpp" #include "custom-types/shared/delegate.hpp" @@ -17,20 +29,138 @@ static modloader::ModInfo modInfo{MOD_ID, VERSION, 1}; using namespace UnityEngine; +using namespace websocketpp; + +static Camera* mainCamera; +static QRUE::CameraStreamer* streamer; + +typedef websocketpp::server WebSocketServer; +static std::unique_ptr streamSocketHandler; +static std::set> connections; + +RenderTexture* CreateCameraTexture(int w, int h) { + auto texture = RenderTexture::New_ctor(w, h, 24, RenderTextureFormat::Default, RenderTextureReadWrite::Default); + Object::DontDestroyOnLoad(texture); + texture->wrapMode = TextureWrapMode::Clamp; + texture->filterMode = FilterMode::Bilinear; + texture->Create(); + return texture; +} + +inline UnityEngine::Matrix4x4 MatrixTranslate(UnityEngine::Vector3 const& vector) { + UnityEngine::Matrix4x4 result; + result.m00 = 1; + result.m01 = 0; + result.m02 = 0; + result.m03 = vector.x; + result.m10 = 0; + result.m11 = 1; + result.m12 = 0; + result.m13 = vector.y; + result.m20 = 0; + result.m21 = 0; + result.m22 = 1; + result.m23 = vector.z; + result.m30 = 0; + result.m31 = 0; + result.m32 = 0; + result.m33 = 1; + return result; +} void onSceneLoad(SceneManagement::Scene scene, SceneManagement::LoadSceneMode) { + if (auto main = Camera::get_main()) { + main->set_enabled(false); + mainCamera = main; + } + + LOG_DEBUG("scene load, main camera is {}", fmt::ptr(mainCamera)); + static bool loaded; if (!scene.IsValid() || loaded) return; loaded = true; - IL2CPP_CATCH_HANDLER(auto go = UnityEngine::GameObject::New_ctor("QuestRUE"); UnityEngine::Object::DontDestroyOnLoad(go); - go->AddComponent();) -} + auto go = GameObject::New_ctor("QuestRUE"); + Object::DontDestroyOnLoad(go); + go->AddComponent(); + + auto cam = go->AddComponent(); -std::string_view GetDataPath() { - static std::string s(getDataDir(modInfo)); - return s; + int w = 1080; + int h = 720; + + cam->set_stereoTargetEye(StereoTargetEyeMask::None); + cam->set_backgroundColor({0, 0, 0, 0}); + cam->set_aspect(w / (float) h); + cam->set_fieldOfView(80); + cam->set_pixelRect({0, 0, (float) w, (float) h}); + cam->set_rect({0, 0, 1, 1}); + +#ifdef UNITY_2021 + auto forwardMult = UnityEngine::Vector3::op_Multiply(UnityEngine::Vector3::get_forward(), -49999.5); + auto mat = UnityEngine::Matrix4x4::Ortho(-99999, 99999, -99999, 99999, 0.001, 99999); + mat = UnityEngine::Matrix4x4::op_Multiply(mat, MatrixTranslate(forwardMult)); + mat = UnityEngine::Matrix4x4::op_Multiply(mat, cam->worldToCameraMatrix); + cam->cullingMatrix = mat; +#endif + + auto tex = CreateCameraTexture(w, h); + cam->set_targetTexture(tex); + + streamer = go->AddComponent(); +#ifdef UNITY_2021 + streamer->texture = (uintptr_t) tex->GetNativeTexturePtr().m_value.convert(); +#else + streamer->texture = (uintptr_t) tex->GetNativeTexturePtr(); +#endif + streamer->onOutputFrame = [](uint8_t* data, size_t length) { + if (!streamSocketHandler) + return; + for (auto const& hdl : connections) { + try { + // LOG_DEBUG("sending {} data", length); + streamSocketHandler->send(hdl, (void*) data, length, frame::opcode::value::BINARY); + } catch (exception const& e) { + LOG_ERROR("send failed: {}", e.what()); + } + } + }; + + streamSocketHandler = std::make_unique(); + streamSocketHandler->set_access_channels(log::alevel::none); + streamSocketHandler->set_error_channels(log::elevel::none); + + streamSocketHandler->init_asio(); + streamSocketHandler->set_reuse_addr(true); + streamSocketHandler->set_open_handler([](websocketpp::connection_hdl hdl) { + scheduleFunction([hdl]() { + connections.clear(); + streamer->Stop(); + connections.insert(hdl); + LOG_INFO("Connected {} status: connected", hdl.lock().get()); + }); + }); + streamSocketHandler->set_close_handler([](websocketpp::connection_hdl hdl) { + scheduleFunction([hdl]() { + streamer->Stop(); + connections.erase(hdl); + LOG_INFO("Connected {} status: disconnected", hdl.lock().get()); + }); + }); + streamSocketHandler->set_message_handler([](websocketpp::connection_hdl hdl, WebSocketServer::message_ptr msg) { + scheduleFunction([packet = std::string(msg->get_payload().data(), msg->get_payload().size())]() { + LOG_DEBUG("got packet {}", packet); + if (packet == "start") + streamer->Init(1080, 720, 1000000); + }); + }); + + lib::asio::error_code ec; + streamSocketHandler->listen(lib::asio::ip::tcp::v4(), 3307, ec); + + streamSocketHandler->start_accept(); + std::thread([]() { streamSocketHandler->run(); }).detach(); } extern "C" void setup(CModInfo* info) { @@ -39,50 +169,25 @@ extern "C" void setup(CModInfo* info) { *info = modInfo.to_c(); - auto dataPath = GetDataPath(); - if (!direxists(dataPath)) - mkpath(dataPath); - -#ifdef BEAT_SABER setenv("QuestRUE", "", 0); -#endif LOG_INFO("Completed setup!"); } #ifdef BEAT_SABER -#include "UnityEngine/Camera.hpp" +#include "GlobalNamespace/BloomPrePass.hpp" +#include "GlobalNamespace/DefaultScenesTransitionsFromInit.hpp" +#include "GlobalNamespace/MainCamera.hpp" +#include "GlobalNamespace/UIKeyboardManager.hpp" +#include "GlobalNamespace/VRPlatformUtils.hpp" +#include "UnityEngine/Input.hpp" +#include "VRUIControls/ButtonState.hpp" +#include "VRUIControls/MouseButtonEventData.hpp" +#include "VRUIControls/MouseState.hpp" +#include "VRUIControls/VRInputModule.hpp" using namespace GlobalNamespace; -void EnableFPFC() { - LOG_INFO("Enabling camera controller"); - - auto cam = UnityEngine::Camera::get_main(); - if (!cam) - return; - - auto go = cam->get_gameObject(); - if (auto existing = go->GetComponent()) - existing->set_enabled(true); - else - go->AddComponent(); -} - -void DisableFPFC() { - auto cam = UnityEngine::Camera::get_main(); - if (!cam) - return; - - auto go = cam->get_gameObject(); - bool enabledState = fpfcEnabled; - if (auto existing = go->GetComponent()) - existing->set_enabled(false); - fpfcEnabled = enabledState; -} - -#include "GlobalNamespace/DefaultScenesTransitionsFromInit.hpp" - MAKE_HOOK_MATCH( DefaultScenesTransitionsFromInit_TransitionToNextScene, &DefaultScenesTransitionsFromInit::TransitionToNextScene, @@ -95,54 +200,6 @@ MAKE_HOOK_MATCH( DefaultScenesTransitionsFromInit_TransitionToNextScene(self, true, goStraightToEditor, goToRecordingToolScene); } -#include "GlobalNamespace/GameScenesManager.hpp" -#include "System/Action_1.hpp" -#include "Zenject/DiContainer.hpp" - -MAKE_HOOK_MATCH( - GameScenesManager_ScenesTransitionCoroutine, - &GameScenesManager::ScenesTransitionCoroutine, - System::Collections::IEnumerator*, - GameScenesManager* self, - GlobalNamespace::ScenesTransitionSetupDataSO* newScenesTransitionSetupData, - System::Collections::Generic::List_1* scenesToPresent, - GlobalNamespace::__GameScenesManager__ScenePresentType presentType, - System::Collections::Generic::List_1* scenesToDismiss, - GlobalNamespace::__GameScenesManager__SceneDismissType dismissType, - float_t minDuration, - System::Action* afterMinDurationCallback, - System::Action_1* extraBindingsCallback, - System::Action_1* finishCallback -) { - DisableFPFC(); - - finishCallback = (System::Action_1*) System::MulticastDelegate::Combine( - finishCallback, - custom_types::MakeDelegate*>((std::function) [](Zenject::DiContainer*) { - if (fpfcEnabled) - EnableFPFC(); - }) - ); - - return GameScenesManager_ScenesTransitionCoroutine( - self, - newScenesTransitionSetupData, - scenesToPresent, - presentType, - scenesToDismiss, - dismissType, - minDuration, - afterMinDurationCallback, - extraBindingsCallback, - finishCallback - ); -} - -#include "VRUIControls/ButtonState.hpp" -#include "VRUIControls/MouseButtonEventData.hpp" -#include "VRUIControls/MouseState.hpp" -#include "VRUIControls/VRInputModule.hpp" - static bool clickedLastFrame = false; MAKE_HOOK_MATCH( @@ -172,9 +229,6 @@ MAKE_HOOK_MATCH( return ret; } -#include "GlobalNamespace/VRPlatformUtils.hpp" -#include "UnityEngine/Input.hpp" - MAKE_HOOK_MATCH( VRPlatformUtils_GetAnyJoystickMaxAxisDefaultImplementation, &VRPlatformUtils::GetAnyJoystickMaxAxisDefaultImplementation, @@ -187,8 +241,6 @@ MAKE_HOOK_MATCH( return ret; } -#include "GlobalNamespace/UIKeyboardManager.hpp" - MAKE_HOOK_MATCH(UIKeyboardManager_OpenKeyboardFor, &UIKeyboardManager::OpenKeyboardFor, void, UIKeyboardManager* self, HMUI::InputFieldView* input) { UIKeyboardManager_OpenKeyboardFor(self, input); @@ -203,24 +255,30 @@ MAKE_HOOK_MATCH(UIKeyboardManager_CloseKeyboard, &UIKeyboardManager::CloseKeyboa } #ifdef UNITY_2021 -#include "UnityEngine/Pose.hpp" -#include "UnityEngine/SpatialTracking/PoseDataFlags.hpp" -#include "UnityEngine/SpatialTracking/TrackedPoseDriver.hpp" +MAKE_HOOK_MATCH(MainCamera_Awake, &MainCamera::Awake, void, MainCamera* self) { -MAKE_HOOK_MATCH( - TrackedPoseDriver_GetPoseData, - &SpatialTracking::TrackedPoseDriver::GetPoseData, - SpatialTracking::PoseDataFlags, - SpatialTracking::TrackedPoseDriver* self, - SpatialTracking::TrackedPoseDriver::DeviceType device, - SpatialTracking::TrackedPoseDriver::TrackedPose poseSource, - ByRef resultPose -) { - if (fpfcEnabled) { - resultPose = Pose(fpfcPos, Quaternion::Euler(fpfcRot)); - return (int) SpatialTracking::PoseDataFlags::Position | (int) SpatialTracking::PoseDataFlags::Rotation; - } else - return TrackedPoseDriver_GetPoseData(self, device, poseSource, resultPose); + MainCamera_Awake(self); + + if (!self->_camera || !streamer) + return; + mainCamera = self->_camera; + self->_camera->enabled = false; + + auto cam = streamer->GetComponent(); + auto mainBloom = self->_camera->GetComponent(); + if (!cam || !mainBloom) + return; + + cam->gameObject->SetActive(false); + auto bloom = cam->gameObject->AddComponent(); + if (!bloom) + bloom = cam->GetComponent(); + bloom->_bloomPrePassEffectContainer = mainBloom->_bloomPrePassEffectContainer; + bloom->_bloomPrepassRenderer = mainBloom->_bloomPrepassRenderer; + bloom->_bloomPrePassRenderData = mainBloom->_bloomPrePassRenderData; + bloom->_mode = mainBloom->_mode; + bloom->_renderData = mainBloom->_renderData; + cam->gameObject->SetActive(true); } #endif #endif @@ -237,9 +295,8 @@ extern "C" void load() { INSTALL_HOOK(logger, VRInputModule_GetMousePointerEventData); INSTALL_HOOK(logger, UIKeyboardManager_OpenKeyboardFor); INSTALL_HOOK(logger, UIKeyboardManager_CloseKeyboard); - INSTALL_HOOK(logger, GameScenesManager_ScenesTransitionCoroutine); #ifdef UNITY_2021 - INSTALL_HOOK(logger, TrackedPoseDriver_GetPoseData); + INSTALL_HOOK(logger, MainCamera_Awake); #endif LOG_INFO("Installed hooks!"); #endif diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 1006f87..206350f 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -131,6 +131,15 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -2194,6 +2203,7 @@ dependencies = [ "tauri", "tauri-build", "tauri-plugin-websocket", + "tauri-plugin-window-state", "tokio", "tokio-util", ] @@ -3008,7 +3018,7 @@ dependencies = [ [[package]] name = "tauri-plugin-websocket" version = "0.0.0" -source = "git+https://github.com/tauri-apps/plugins-workspace?branch=v1#2e627bc0b709055c564fe40d670c008a7771471b" +source = "git+https://github.com/tauri-apps/plugins-workspace?branch=v1#0a8484c52d74215bc4cbcc7fcb58e376f4fcd1f7" dependencies = [ "futures-util", "http 1.1.0", @@ -3022,6 +3032,20 @@ dependencies = [ "tokio-tungstenite", ] +[[package]] +name = "tauri-plugin-window-state" +version = "0.1.1" +source = "git+https://github.com/tauri-apps/plugins-workspace?branch=v1#0a8484c52d74215bc4cbcc7fcb58e376f4fcd1f7" +dependencies = [ + "bincode", + "bitflags 2.6.0", + "log", + "serde", + "serde_json", + "tauri", + "thiserror", +] + [[package]] name = "tauri-runtime" version = "0.14.5" @@ -3259,9 +3283,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.23.1" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" dependencies = [ "futures-util", "log", @@ -3419,9 +3443,9 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" dependencies = [ "byteorder", "bytes", @@ -3730,7 +3754,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 96ef2b2..89d226e 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -14,13 +14,14 @@ tauri-build = { version = "1.5", features = [] } [dependencies] tauri-plugin-websocket = { git = "https://github.com/tauri-apps/plugins-workspace", branch = "v1" } +tauri-plugin-window-state = { git = "https://github.com/tauri-apps/plugins-workspace", branch = "v1" } tokio = { version = "1", features = ["full"] } tokio-util = { version = "0.7", features = ["full"] } bytes = "*" futures = "*" anyhow = "1.0" serde_json = "1.0" -tauri = { version = "1.6", features = [ "window-all", "shell-execute", "shell-open", "shell-open-api", "devtools"] } +tauri = { version = "1.6", features = [ "fs-all", "window-all", "shell-execute", "shell-open", "shell-open-api", "devtools"] } serde = { version = "1.0", features = ["derive"] } log = "0.4" env_logger = "0.10" diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index 058d8a3..d43b610 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -7,6 +7,7 @@ async fn main() -> anyhow::Result<()> { tauri::Builder::default() .plugin(tauri_plugin_websocket::init()) + .plugin(tauri_plugin_window_state::Builder::default().build()) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 4bf5d45..7de4518 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -52,7 +52,7 @@ "fullscreen": false, "resizable": true, "maximized": true, - "title": "quest-rue" + "title": "Quest RUE" } ] } diff --git a/src/Stream.tsx b/src/Stream.tsx new file mode 100644 index 0000000..b9584f0 --- /dev/null +++ b/src/Stream.tsx @@ -0,0 +1,129 @@ +import { render } from "solid-js/web"; +import "./styles.css"; +import "solid-devtools"; +import { createEffect, createSignal } from "solid-js"; +import WebSocketRS from "tauri-plugin-websocket-api"; +import { isTauri } from "./misc/dev"; + +// todo: refactor websocket.ts +function connect(url: string, worker: Worker) { + if (isTauri()) { + WebSocketRS.connect(url) + .then((connected) => { + console.log("socket connected"); + connected.addListener((arg) => { + switch (arg.type) { + case undefined: // error: close without handshake + case "Close": { + console.log("socket closed"); + break; + } + case "Binary": { + worker.postMessage({ + type: "data", + val: new Uint8Array(arg.data), + }); + break; + } + } + }); + connected.send("start"); + }) + .catch((err) => { + console.log("failure connecting socket", err); + }); + } else { + const socket = new WebSocket(url); + socket.binaryType = "arraybuffer"; + socket.onopen = () => { + console.log("socket connected"); + socket.send("start"); + }; + socket.onmessage = (event) => { + worker.postMessage({ + type: "data", + val: new Uint8Array(event.data), + }); + }; + } +} + +function Stream() { + const decoder = new Worker( + new URL("misc/decoder_worker.ts", import.meta.url), + ); + + let canvas: HTMLCanvasElement | undefined; + + createEffect(() => { + const offscreen = canvas!.transferControlToOffscreen(); + decoder.postMessage({ type: "context", val: offscreen }, [offscreen]); + }); + + connect("ws://localhost:3307", decoder); + + const [pointerLocked, setPointerLocked] = createSignal(false); + + const onMouseMove = (event: MouseEvent) => { + console.log("mouse move", event.movementX, event.movementY); + }; + + const onMouseDown = (event: MouseEvent) => { + console.log("mouse down", event.button, event.button === 0); + }; + + const onMouseUp = (event: MouseEvent) => { + console.log("mouse up", event.button, event.button === 0); + }; + + const onKeyDown = (event: KeyboardEvent) => { + console.log("key down", event.key); + }; + + const onKeyUp = (event: KeyboardEvent) => { + console.log("key up", event.key); + }; + + document.addEventListener("pointerlockchange", () => { + setPointerLocked( + canvas !== undefined && document.pointerLockElement === canvas, + ); + }); + + createEffect(() => { + if (pointerLocked()) { + document.addEventListener("mousemove", onMouseMove); + document.addEventListener("mousedown", onMouseDown); + document.addEventListener("mouseup", onMouseUp); + document.addEventListener("keydown", onKeyDown); + document.addEventListener("keyup", onKeyUp); + } else { + document.removeEventListener("mousemove", onMouseMove); + document.removeEventListener("mousedown", onMouseDown); + document.removeEventListener("mouseup", onMouseUp); + document.removeEventListener("keydown", onKeyDown); + document.removeEventListener("keyup", onKeyUp); + } + }); + + return ( +
+ { + canvas?.requestPointerLock(); + }} + /> +
+ ); +} + +render(() => , document.getElementById("root") as HTMLElement); diff --git a/src/index.tsx b/src/index.tsx index 8fc7609..88df640 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -1,4 +1,3 @@ -/* @refresh reload */ import { render } from "solid-js/web"; import "@thisbeyond/solid-select/style.css"; diff --git a/src/misc/decoder_worker.ts b/src/misc/decoder_worker.ts new file mode 100644 index 0000000..3a8398e --- /dev/null +++ b/src/misc/decoder_worker.ts @@ -0,0 +1,68 @@ +let decoderError = false; +let wroteSps = false; +let spsPacket: Uint8Array | undefined; +let canvasContext: OffscreenCanvasRenderingContext2D | undefined; +let decoder: VideoDecoder | undefined; + +const config: VideoDecoderConfig = { + codec: "avc1.42000a", + codedWidth: 1080, + codedHeight: 720, + optimizeForLatency: true, + hardwareAcceleration: "prefer-software", +}; + +const init: VideoDecoderInit = { + output: (frame) => { + canvasContext?.drawImage(frame, 0, 0); + frame.close(); + }, + error: (error) => { + console.error("decoder error", error); + decoderError = true; + }, +}; + +VideoDecoder.isConfigSupported(config).then((support) => { + if (!support.supported) { + console.error("config unsupported!", support); + return; + } + decoder = new VideoDecoder(init); + decoder.configure(config); +}); + +self.onmessage = (event: MessageEvent) => { + const data = event.data as + | { type: "context"; val: OffscreenCanvas } + | { type: "data"; val: Uint8Array }; + + if (data.type === "context") canvasContext = data.val.getContext("2d")!; + else { + try { + if (decoder === undefined || decoderError) return; + let array = data.val; + if (array[0] !== 0 || array[1] !== 0 || array[2] !== 0 || array[3] !== 1) + console.warn("invalid header", array); + if (array[4] === 103) { + spsPacket = new Uint8Array(array); + return; + } + if (decoder.decodeQueueSize < 5) { + if (spsPacket !== undefined && !wroteSps && array[4] === 101) { + console.log("combining sps with normal packet"); + array = new Uint8Array([...spsPacket!, ...array]); + wroteSps = true; + } + const chunk = new EncodedVideoChunk({ + data: array, + timestamp: 0, + type: array[4] === 101 ? "key" : "delta", + }); + decoder.decode(chunk); + } else console.log("skipping chunk!"); + } catch (error) { + console.error("decoding error", error); + } + } +}; diff --git a/stream.html b/stream.html new file mode 100644 index 0000000..45978ef --- /dev/null +++ b/stream.html @@ -0,0 +1,18 @@ + + + + + + + + + QRUE Stream + + + + +
+ + + +