diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 71cd3d4c7b..18aa3e8dd7 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -213,6 +213,11 @@ cc_inspect_values( USE_REMOTE_LOG ) +if(USE_XR) + if(USE_XR_REMOTE_PREVIEW) + set(USE_WEBSOCKET_SERVER ON) + endif() +endif() include(${EXTERNAL_ROOT}/CMakeLists.txt) @@ -2122,6 +2127,13 @@ if(USE_XR) ) endif() +if(USE_XR_REMOTE_PREVIEW) + cocos_source_files( + NO_WERROR cocos/xr/XRRemotePreviewManager.h + cocos/xr/XRRemotePreviewManager.cpp + ) +endif() + if(USE_MIDDLEWARE) cocos_source_files( cocos/editor-support/IOBuffer.cpp @@ -2894,6 +2906,7 @@ function(cc_apply_definations target) $,CC_USE_WEBVIEW=1,CC_USE_WEBVIEW=0> $,CC_USE_AUDIO=1,CC_USE_AUDIO=0> $,CC_USE_XR=1,CC_USE_XR=0> + $,CC_USE_XR_REMOTE_PREVIEW=1,CC_USE_XR_REMOTE_PREVIEW=0> $,CC_USE_SOCKET=1,CC_USE_SOCKET=0> $,CC_USE_WEBSOCKET_SERVER=1,CC_USE_WEBSOCKET_SERVER=0> $,CC_USE_EDITBOX=1,CC_USE_EDITBOX=0> diff --git a/native/cocos/platform/interfaces/modules/IXRInterface.h b/native/cocos/platform/interfaces/modules/IXRInterface.h index 851b7dcc92..365a5d5c96 100644 --- a/native/cocos/platform/interfaces/modules/IXRInterface.h +++ b/native/cocos/platform/interfaces/modules/IXRInterface.h @@ -283,6 +283,7 @@ public: * @return */ virtual ccstd::vector getHMDViewPosition(uint32_t eye, int trackingType) = 0; + /** * @en get xr view projection data * @zh 获取xr双眼投影矩阵数据 @@ -293,6 +294,14 @@ public: */ virtual ccstd::vector getXRViewProjectionData(uint32_t eye, float near, float far) = 0; + /** + * @en get xr eye's fov + * @zh 获取xr双眼视场角 + * @param eye + * @return + */ + virtual ccstd::vector getXREyeFov(uint32_t eye) = 0; + // renderwindow /** * @en get renderwindow's xreye type diff --git a/native/cocos/platform/interfaces/modules/XRCommon.h b/native/cocos/platform/interfaces/modules/XRCommon.h index b0ea4aba5c..c7718e1fa1 100644 --- a/native/cocos/platform/interfaces/modules/XRCommon.h +++ b/native/cocos/platform/interfaces/modules/XRCommon.h @@ -52,24 +52,42 @@ enum class XRVendor { }; enum class XRConfigKey { - MULTI_SAMPLES, - RENDER_SCALE, - SESSION_RUNNING, - INSTANCE_CREATED, - VK_QUEUE_FAMILY_INDEX, - METRICS_STATE, - VIEW_COUNT, - SWAPCHAIN_WIDTH, - SWAPCHAIN_HEIGHT, - SWAPCHAIN_FORMAT, - MULTITHREAD_MODE, - LOGIC_THREAD_ID, - RENDER_THREAD_ID, - DEVICE_VENDOR, - RUNTIME_VERSION, - PRESENT_ENABLE, - RENDER_EYE_FRAME_LEFT, - RENDER_EYE_FRAME_RIGHT + MULTI_SAMPLES = 0, + RENDER_SCALE = 1, + SESSION_RUNNING = 2, + INSTANCE_CREATED = 3, + VK_QUEUE_FAMILY_INDEX = 4, + METRICS_STATE = 5, + VIEW_COUNT = 6, + SWAPCHAIN_WIDTH = 7, + SWAPCHAIN_HEIGHT = 8, + SWAPCHAIN_FORMAT = 9, + MULTITHREAD_MODE = 10, + LOGIC_THREAD_ID = 11, + RENDER_THREAD_ID = 12, + DEVICE_VENDOR = 13, + RUNTIME_VERSION = 14, + PRESENT_ENABLE = 15, + RENDER_EYE_FRAME_LEFT = 16, + RENDER_EYE_FRAME_RIGHT = 17, + FEATURE_PASSTHROUGH= 18, + IMAGE_TRACKING = 19, + IMAGE_TRACKING_CANDIDATEIMAGE = 20, + IMAGE_TRACKING_DATA = 21, + IMAGE_TRACKING_SUPPORT_STATUS = 22, + HIT_TESTING = 23, + HIT_TESTING_DATA = 24, + HIT_TESTING_SUPPORT_STATUS = 25, + PLANE_DETECTION = 26, + PLANE_DETECTION_DATA = 27, + PLANE_DETECTION_SUPPORT_STATUS = 28, + SPATIAL_ANCHOR = 29, + SPATIAL_ANCHOR_DATA = 30, + SPATIAL_ANCHOR_SUPPORT_STATUS = 31, + HAND_TRACKING = 32, + HAND_TRACKING_DATA = 33, + HAND_TRACKING_SUPPORT_STATUS = 34, + MAX_COUNT }; enum class XRConfigValueType { @@ -310,6 +328,19 @@ struct XRSwapchain { uint32_t eye = 0; }; +struct XRTrackingImageData { + std::string friendlyName; + uint32_t id; + uint8_t *buffer; + uint32_t bufferSize; + float physicalWidth; + float physicalHeight; + uint32_t pixelSizeWidth; + uint32_t pixelSizeHeight; + float posePosition[3]; + float poseQuaternion[4]; +}; + #define GraphicsApiOpenglES "OpenGLES" #define GraphicsApiVulkan_1_0 "Vulkan1" #define GraphicsApiVulkan_1_1 "Vulkan2" diff --git a/native/cocos/platform/java/modules/XRInterface.cpp b/native/cocos/platform/java/modules/XRInterface.cpp index e584253d89..d920d9dd68 100644 --- a/native/cocos/platform/java/modules/XRInterface.cpp +++ b/native/cocos/platform/java/modules/XRInterface.cpp @@ -28,6 +28,7 @@ #include #include #include +#include "base/StringUtil.h" #include "android/AndroidPlatform.h" #include "base/Log.h" #include "base/Macros.h" @@ -49,14 +50,39 @@ #endif #include "application/ApplicationManager.h" #include "base/threading/MessageQueue.h" +#include "platform/Image.h" // print log const bool IS_ENABLE_XR_LOG = false; namespace cc { -ControllerEvent controllerEvent; -static se::Object *jsPoseEventArray = nullptr; -static void dispatchGamepadEventInternal(const xr::XRControllerEvent &xrControllerEvent) { +const static ccstd::unordered_map CLICK_TYPE_TO_KEY_CODE = { + {xr::XRClick::Type::MENU, StickKeyCode::MENU}, + {xr::XRClick::Type::TRIGGER_LEFT, StickKeyCode::TRIGGER_LEFT}, + {xr::XRClick::Type::SHOULDER_LEFT, StickKeyCode::L1}, + {xr::XRClick::Type::THUMBSTICK_LEFT, StickKeyCode::L3}, + {xr::XRClick::Type::X, StickKeyCode::Y}, + {xr::XRClick::Type::Y, StickKeyCode::X}, + {xr::XRClick::Type::TRIGGER_RIGHT, StickKeyCode::TRIGGER_RIGHT}, + {xr::XRClick::Type::SHOULDER_RIGHT, StickKeyCode::R1}, + {xr::XRClick::Type::THUMBSTICK_RIGHT, StickKeyCode::R3}, + {xr::XRClick::Type::A, StickKeyCode::B}, + {xr::XRClick::Type::B, StickKeyCode::A}, + {xr::XRClick::Type::HOME, StickKeyCode::UNDEFINE}, + {xr::XRClick::Type::START, StickKeyCode::START}, + {xr::XRClick::Type::DPAD_DOWN, StickKeyCode::Y}, + {xr::XRClick::Type::DPAD_UP, StickKeyCode::Y}, + {xr::XRClick::Type::DPAD_LEFT, StickKeyCode::X}, + {xr::XRClick::Type::DPAD_RIGHT, StickKeyCode::X}}; + +const static ccstd::unordered_map GRAB_TYPE_TO_AXIS_CODE = { + {xr::XRGrab::Type::TRIGGER_LEFT, StickAxisCode::L2}, + {xr::XRGrab::Type::TRIGGER_RIGHT, StickAxisCode::R2}, + {xr::XRGrab::Type::GRIP_LEFT, StickAxisCode::LEFT_GRIP}, + {xr::XRGrab::Type::GRIP_RIGHT, StickAxisCode::RIGHT_GRIP}, +}; + +void XRInterface::dispatchGamepadEventInternal(const xr::XRControllerEvent &xrControllerEvent) { if (xrControllerEvent.xrControllerInfos.empty()) { return; } @@ -71,41 +97,25 @@ static void dispatchGamepadEventInternal(const xr::XRControllerEvent &xrControll switch (xrControllerEvent.xrControllerInfos.at(i)->getXREventType()) { case xr::XREventType::CLICK: { auto *xrClick = static_cast(xrControllerEvent.xrControllerInfos.at(i).get()); + StickKeyCode stickKeyCode = CLICK_TYPE_TO_KEY_CODE.at(xrClick->type); + switch (xrClick->type) { case xr::XRClick::Type::MENU: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::MENU, xrClick->isPress)); - break; case xr::XRClick::Type::TRIGGER_LEFT: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::TRIGGER_LEFT, xrClick->isPress)); - break; case xr::XRClick::Type::SHOULDER_LEFT: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::L1, xrClick->isPress)); - break; case xr::XRClick::Type::THUMBSTICK_LEFT: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::L3, xrClick->isPress)); - break; case xr::XRClick::Type::X: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::Y, xrClick->isPress)); - break; case xr::XRClick::Type::Y: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::X, xrClick->isPress)); - break; case xr::XRClick::Type::TRIGGER_RIGHT: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::TRIGGER_RIGHT, xrClick->isPress)); - break; case xr::XRClick::Type::SHOULDER_RIGHT: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::R1, xrClick->isPress)); - break; case xr::XRClick::Type::THUMBSTICK_RIGHT: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::R3, xrClick->isPress)); - break; case xr::XRClick::Type::A: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::B, xrClick->isPress)); - break; case xr::XRClick::Type::B: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::A, xrClick->isPress)); + case xr::XRClick::Type::START: { + controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(stickKeyCode, xrClick->isPress)); break; - case xr::XRClick::Type::HOME: + } + case xr::XRClick::Type::HOME: { CC_LOG_INFO("[XRInterface] exit when home click in rokid."); #if CC_USE_XR xr::XrEntry::getInstance()->destroyXrInstance(); @@ -113,9 +123,7 @@ static void dispatchGamepadEventInternal(const xr::XRControllerEvent &xrControll #endif CC_CURRENT_APPLICATION_SAFE()->close(); break; - case xr::XRClick::Type::START: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::START, xrClick->isPress)); - break; + } case xr::XRClick::Type::DPAD_UP: controllerInfo->axisInfos.emplace_back(ControllerInfo::AxisInfo(StickAxisCode::Y, xrClick->isPress ? 1.F : 0.F)); break; @@ -149,13 +157,13 @@ static void dispatchGamepadEventInternal(const xr::XRControllerEvent &xrControll } break; case xr::XREventType::GRAB: { auto *xrGrab = static_cast(xrControllerEvent.xrControllerInfos.at(i).get()); + StickAxisCode stickAxisCode = GRAB_TYPE_TO_AXIS_CODE.at(xrGrab->type); switch (xrGrab->type) { case xr::XRGrab::Type::TRIGGER_LEFT: - controllerInfo->axisInfos.emplace_back(ControllerInfo::AxisInfo(StickAxisCode::L2, xrGrab->value)); - break; - case xr::XRGrab::Type::TRIGGER_RIGHT: - controllerInfo->axisInfos.emplace_back(ControllerInfo::AxisInfo(StickAxisCode::R2, xrGrab->value)); + case xr::XRGrab::Type::TRIGGER_RIGHT: { + controllerInfo->axisInfos.emplace_back(ControllerInfo::AxisInfo(stickAxisCode, xrGrab->value)); break; + } default: break; } @@ -166,14 +174,29 @@ static void dispatchGamepadEventInternal(const xr::XRControllerEvent &xrControll } controllerInfo->napdId = 0; // xr only one gamepad connection - controllerEvent.controllerInfos.emplace_back(controllerInfo); - controllerEvent.type = ControllerEvent::Type::GAMEPAD; - events::Controller::broadcast(controllerEvent); - controllerEvent.type = ControllerEvent::Type::UNKNOWN; - controllerEvent.controllerInfos.clear(); + _controllerEvent.controllerInfos.emplace_back(controllerInfo); + _controllerEvent.type = ControllerEvent::Type::GAMEPAD; +#if CC_USE_XR_REMOTE_PREVIEW + if (_xrRemotePreviewManager) { + if (!controllerInfo->buttonInfos.empty()) { + for (const auto &btnInfo : controllerInfo->buttonInfos) { + _xrRemotePreviewManager->sendControllerKeyInfo(btnInfo); + } + } + + if (!controllerInfo->axisInfos.empty()) { + for (const auto &axisInfo : controllerInfo->axisInfos) { + _xrRemotePreviewManager->sendControllerKeyInfo(axisInfo); + } + } + } +#endif + events::Controller::broadcast(_controllerEvent); + _controllerEvent.type = ControllerEvent::Type::UNKNOWN; + _controllerEvent.controllerInfos.clear(); } -static void dispatchHandleEventInternal(const xr::XRControllerEvent &xrControllerEvent) { +void XRInterface::dispatchHandleEventInternal(const xr::XRControllerEvent &xrControllerEvent) { if (xrControllerEvent.xrControllerInfos.empty()) { return; } @@ -190,8 +213,9 @@ static void dispatchHandleEventInternal(const xr::XRControllerEvent &xrControlle switch (xrControllerEvent.xrControllerInfos.at(i)->getXREventType()) { case xr::XREventType::CLICK: { auto *xrClick = static_cast(xrControllerEvent.xrControllerInfos.at(i).get()); + StickKeyCode stickKeyCode = CLICK_TYPE_TO_KEY_CODE.at(xrClick->type); switch (xrClick->type) { - case xr::XRClick::Type::MENU: + case xr::XRClick::Type::MENU: { #if !XR_OEM_SEED controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::MENU, xrClick->isPress)); #else @@ -199,30 +223,18 @@ static void dispatchHandleEventInternal(const xr::XRControllerEvent &xrControlle CC_CURRENT_APPLICATION_SAFE()->close(); #endif break; + } case xr::XRClick::Type::TRIGGER_LEFT: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::TRIGGER_LEFT, xrClick->isPress)); - break; case xr::XRClick::Type::THUMBSTICK_LEFT: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::L3, xrClick->isPress)); - break; case xr::XRClick::Type::X: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::Y, xrClick->isPress)); - break; case xr::XRClick::Type::Y: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::X, xrClick->isPress)); - break; case xr::XRClick::Type::TRIGGER_RIGHT: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::TRIGGER_RIGHT, xrClick->isPress)); - break; case xr::XRClick::Type::THUMBSTICK_RIGHT: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::R3, xrClick->isPress)); - break; case xr::XRClick::Type::A: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::B, xrClick->isPress)); - break; - case xr::XRClick::Type::B: - controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(StickKeyCode::A, xrClick->isPress)); + case xr::XRClick::Type::B: { + controllerInfo->buttonInfos.emplace_back(ControllerInfo::ButtonInfo(stickKeyCode, xrClick->isPress)); break; + } default: break; } @@ -244,19 +256,15 @@ static void dispatchHandleEventInternal(const xr::XRControllerEvent &xrControlle } break; case xr::XREventType::GRAB: { auto *xrGrab = static_cast(xrControllerEvent.xrControllerInfos.at(i).get()); + StickAxisCode stickAxisCode = GRAB_TYPE_TO_AXIS_CODE.at(xrGrab->type); switch (xrGrab->type) { case xr::XRGrab::Type::TRIGGER_LEFT: - controllerInfo->axisInfos.emplace_back(ControllerInfo::AxisInfo(StickAxisCode::L2, xrGrab->value)); - break; case xr::XRGrab::Type::TRIGGER_RIGHT: - controllerInfo->axisInfos.emplace_back(ControllerInfo::AxisInfo(StickAxisCode::R2, xrGrab->value)); - break; case xr::XRGrab::Type::GRIP_LEFT: - controllerInfo->axisInfos.emplace_back(ControllerInfo::AxisInfo(StickAxisCode::LEFT_GRIP, xrGrab->value)); - break; - case xr::XRGrab::Type::GRIP_RIGHT: - controllerInfo->axisInfos.emplace_back(ControllerInfo::AxisInfo(StickAxisCode::RIGHT_GRIP, xrGrab->value)); + case xr::XRGrab::Type::GRIP_RIGHT: { + controllerInfo->axisInfos.emplace_back(ControllerInfo::AxisInfo(stickAxisCode, xrGrab->value)); break; + } default: break; } @@ -277,11 +285,11 @@ static void dispatchHandleEventInternal(const xr::XRControllerEvent &xrControlle jsPose->setProperty("quaternionY", se::Value(xrPose->qy)); jsPose->setProperty("quaternionZ", se::Value(xrPose->qz)); jsPose->setProperty("quaternionW", se::Value(xrPose->qw)); - if (!jsPoseEventArray) { - jsPoseEventArray = se::Object::createArrayObject(0); - jsPoseEventArray->root(); + if (!_jsPoseEventArray) { + _jsPoseEventArray = se::Object::createArrayObject(0); + _jsPoseEventArray->root(); } - jsPoseEventArray->setArrayElement(poseIndex, se::Value(jsPose)); + _jsPoseEventArray->setArrayElement(poseIndex, se::Value(jsPose)); poseIndex++; } break; default: @@ -294,26 +302,40 @@ static void dispatchHandleEventInternal(const xr::XRControllerEvent &xrControlle } if (poseIndex > 0) { - jsPoseEventArray->setProperty("length", se::Value(poseIndex)); + _jsPoseEventArray->setProperty("length", se::Value(poseIndex)); se::ValueArray args; - args.emplace_back(se::Value(jsPoseEventArray)); + args.emplace_back(se::Value(_jsPoseEventArray)); EventDispatcher::doDispatchJsEvent("onHandlePoseInput", args); } if (!controllerInfo->buttonInfos.empty() || !controllerInfo->axisInfos.empty()) { controllerInfo->napdId = 0; // xr only one handle connection - controllerEvent.controllerInfos.emplace_back(controllerInfo); - controllerEvent.type = ControllerEvent::Type::HANDLE; - events::Controller::broadcast(controllerEvent); + _controllerEvent.controllerInfos.emplace_back(controllerInfo); + _controllerEvent.type = ControllerEvent::Type::HANDLE; +#if CC_USE_XR_REMOTE_PREVIEW + if (_xrRemotePreviewManager) { + if (!controllerInfo->buttonInfos.empty()) { + for (const auto &btnInfo : controllerInfo->buttonInfos) { + _xrRemotePreviewManager->sendControllerKeyInfo(btnInfo); + } + } - controllerEvent.type = ControllerEvent::Type::UNKNOWN; - controllerEvent.controllerInfos.clear(); + if (!controllerInfo->axisInfos.empty()) { + for (const auto &axisInfo : controllerInfo->axisInfos) { + _xrRemotePreviewManager->sendControllerKeyInfo(axisInfo); + } + } + } +#endif + events::Controller::broadcast(_controllerEvent); + _controllerEvent.type = ControllerEvent::Type::UNKNOWN; + _controllerEvent.controllerInfos.clear(); } else { - CC_SAFE_DELETE(controllerInfo); + CC_SAFE_DELETE(controllerInfo) } } -static void dispatchHMDEventInternal(const xr::XRControllerEvent &xrControllerEvent) { +void XRInterface::dispatchHMDEventInternal(const xr::XRControllerEvent &xrControllerEvent) { if (xrControllerEvent.xrControllerInfos.empty()) { return; } @@ -337,11 +359,11 @@ static void dispatchHMDEventInternal(const xr::XRControllerEvent &xrControllerEv jsPose->setProperty("quaternionY", se::Value(xrPose->qy)); jsPose->setProperty("quaternionZ", se::Value(xrPose->qz)); jsPose->setProperty("quaternionW", se::Value(xrPose->qw)); - if (!jsPoseEventArray) { - jsPoseEventArray = se::Object::createArrayObject(0); - jsPoseEventArray->root(); + if (!_jsPoseEventArray) { + _jsPoseEventArray = se::Object::createArrayObject(0); + _jsPoseEventArray->root(); } - jsPoseEventArray->setArrayElement(poseIndex, se::Value(jsPose)); + _jsPoseEventArray->setArrayElement(poseIndex, se::Value(jsPose)); poseIndex++; } break; default: @@ -351,27 +373,29 @@ static void dispatchHMDEventInternal(const xr::XRControllerEvent &xrControllerEv } if (poseIndex > 0) { - jsPoseEventArray->setProperty("length", se::Value(poseIndex)); + _jsPoseEventArray->setProperty("length", se::Value(poseIndex)); se::ValueArray args; - args.emplace_back(se::Value(jsPoseEventArray)); + args.emplace_back(se::Value(_jsPoseEventArray)); EventDispatcher::doDispatchJsEvent("onHMDPoseInput", args); } } xr::XRVendor XRInterface::getVendor() { #if CC_USE_XR - return (xr::XRVendor)xr::XrEntry::getInstance()->getXRConfig(cc::xr::XRConfigKey::DEVICE_VENDOR).getInt(); -#endif + return static_cast(xr::XrEntry::getInstance()->getXRConfig(cc::xr::XRConfigKey::DEVICE_VENDOR).getInt()); +#else return xr::XRVendor::MONADO; +#endif } xr::XRConfigValue XRInterface::getXRConfig(xr::XRConfigKey key) { #if CC_USE_XR return xr::XrEntry::getInstance()->getXRConfig(key); -#endif +#else CC_UNUSED_PARAM(key); cc::xr::XRConfigValue configValue; return configValue; +#endif } void XRInterface::setXRConfig(xr::XRConfigKey key, xr::XRConfigValue value) { @@ -387,25 +411,38 @@ void XRInterface::setXRConfig(xr::XRConfigKey key, xr::XRConfigValue value) { uint32_t XRInterface::getRuntimeVersion() { #if CC_USE_XR return xr::XrEntry::getInstance()->getXRConfig(cc::xr::XRConfigKey::RUNTIME_VERSION).getInt(); -#endif +#else return 1; +#endif } void XRInterface::initialize(void *javaVM, void *activity) { #if CC_USE_XR CC_LOG_INFO("[XR] initialize vm.%p,aty.%p | %d", javaVM, activity, (int)gettid()); xr::XrEntry::getInstance()->initPlatformData(javaVM, activity); - xr::XrEntry::getInstance()->setGamepadCallback(&dispatchGamepadEventInternal); - xr::XrEntry::getInstance()->setHandleCallback(&dispatchHandleEventInternal); - xr::XrEntry::getInstance()->setHMDCallback(&dispatchHMDEventInternal); - xr::XrEntry::getInstance()->setXRConfig(xr::XRConfigKey::LOGIC_THREAD_ID, (int)gettid()); + xr::XrEntry::getInstance()->setGamepadCallback(std::bind(&XRInterface::dispatchGamepadEventInternal, this, std::placeholders::_1)); + xr::XrEntry::getInstance()->setHandleCallback(std::bind(&XRInterface::dispatchHandleEventInternal, this, std::placeholders::_1)); + xr::XrEntry::getInstance()->setHMDCallback(std::bind(&XRInterface::dispatchHMDEventInternal, this, std::placeholders::_1)); + xr::XrEntry::getInstance()->setXRConfig(xr::XRConfigKey::LOGIC_THREAD_ID, static_cast(gettid())); xr::XrEntry::getInstance()->setXRConfigCallback([this](xr::XRConfigKey key, xr::XRConfigValue value) { if (IS_ENABLE_XR_LOG) CC_LOG_INFO("XRConfigCallback.%d", key); if (key == xr::XRConfigKey::RENDER_EYE_FRAME_LEFT || key == xr::XRConfigKey::RENDER_EYE_FRAME_RIGHT) { - if (value.getInt() == 0) + if (value.getInt() == 0) { this->beginRenderEyeFrame(key == xr::XRConfigKey::RENDER_EYE_FRAME_LEFT ? 0 : 1); - if (value.getInt() == 1) + } + + if (value.getInt() == 1) { this->endRenderEyeFrame(key == xr::XRConfigKey::RENDER_EYE_FRAME_LEFT ? 0 : 1); + } + } else if(key == xr::XRConfigKey::IMAGE_TRACKING_CANDIDATEIMAGE && value.isString()) { + if(!_gThreadPool) { + _gThreadPool = LegacyThreadPool::newSingleThreadPool(); + } + + std::string imageInfo = value.getString(); + _gThreadPool->pushTask([imageInfo, this](int /*tid*/) { + this->loadAssetsImage(imageInfo); + }); } }); #if XR_OEM_PICO @@ -415,6 +452,10 @@ void XRInterface::initialize(void *javaVM, void *activity) { #endif xr::XrEntry::getInstance()->createXrInstance(graphicsApiName.c_str()); #endif + + #if CC_USE_XR_REMOTE_PREVIEW + _xrRemotePreviewManager = new XRRemotePreviewManager(); + #endif #else CC_UNUSED_PARAM(javaVM); CC_UNUSED_PARAM(activity); @@ -427,6 +468,11 @@ void XRInterface::onRenderPause() { if (!_renderPaused) { _renderPaused = true; _renderResumed = false; + #if CC_USE_XR_REMOTE_PREVIEW + if (_xrRemotePreviewManager) { + _xrRemotePreviewManager->pause(); + } + #endif CC_LOG_INFO("[XR] onRenderPause"); xr::XrEntry::getInstance()->pauseXrInstance(); } @@ -440,6 +486,11 @@ void XRInterface::onRenderResume() { _renderPaused = false; CC_LOG_INFO("[XR] onRenderResume"); xr::XrEntry::getInstance()->resumeXrInstance(); + #if CC_USE_XR_REMOTE_PREVIEW + if (_xrRemotePreviewManager) { + _xrRemotePreviewManager->resume(); + } + #endif } #endif } @@ -449,11 +500,16 @@ void XRInterface::onRenderDestroy() { CC_LOG_INFO("[XR] onRenderDestroy"); xr::XrEntry::getInstance()->destroyXrInstance(); xr::XrEntry::destroyInstance(); - if (jsPoseEventArray != nullptr) { - jsPoseEventArray->unroot(); - jsPoseEventArray->decRef(); - jsPoseEventArray = nullptr; + if (_jsPoseEventArray != nullptr) { + _jsPoseEventArray->unroot(); + _jsPoseEventArray->decRef(); + _jsPoseEventArray = nullptr; } + #if CC_USE_XR_REMOTE_PREVIEW + if (_xrRemotePreviewManager) { + _xrRemotePreviewManager->stop(); + } + #endif #endif } // render thread lifecycle @@ -462,8 +518,8 @@ void XRInterface::onRenderDestroy() { void XRInterface::preGFXDeviceInitialize(gfx::API gfxApi) { #if CC_USE_XR CC_LOG_INFO("[XR] preGFXDeviceInitialize.api.%d | Multi Thread.%d", gfxApi, gfx::DeviceAgent::getInstance() ? 1 : 0); - setXRConfig(xr::XRConfigKey::MULTITHREAD_MODE, gfx::DeviceAgent::getInstance() ? true : false); - xr::XrEntry::getInstance()->setXRConfig(xr::XRConfigKey::RENDER_THREAD_ID, (int)gettid()); + setXRConfig(xr::XRConfigKey::MULTITHREAD_MODE, gfx::DeviceAgent::getInstance() != nullptr); + xr::XrEntry::getInstance()->setXRConfig(xr::XRConfigKey::RENDER_THREAD_ID, static_cast(gettid())); if (gfxApi == gfx::API::GLES3 || gfxApi == gfx::API::VULKAN) { #if !XR_OEM_PICO @@ -501,7 +557,7 @@ const xr::XRSwapchain &XRInterface::doGFXDeviceAcquire(gfx::API gfxApi) { #if CC_USE_XR // CC_LOG_INFO("[XR] doGFXDeviceAcquire.api.%d", gfxApi); if (gfxApi == gfx::API::GLES3 || gfxApi == gfx::API::VULKAN) { - return xr::XrEntry::getInstance()->acquireXrSwapchain((uint32_t)gfxApi); + return xr::XrEntry::getInstance()->acquireXrSwapchain(static_cast(gfxApi)); } #else CC_UNUSED_PARAM(gfxApi); @@ -516,8 +572,9 @@ bool XRInterface::isGFXDeviceNeedsPresent(gfx::API gfxApi) { // if (gfxApi == gfx::API::GLES3 || gfxApi == gfx::API::VULKAN) { // } return xr::XrEntry::getInstance()->getXRConfig(cc::xr::XRConfigKey::PRESENT_ENABLE).getBool(); -#endif +#else return true; +#endif } void XRInterface::postGFXDevicePresent(gfx::API gfxApi) { @@ -541,7 +598,7 @@ void XRInterface::createXRSwapchains() { xr::XrEntry::getInstance()->setXRConfig(xr::XRConfigKey::RENDER_THREAD_ID, (int)gettid()); JniHelper::getEnv(); xr::XrEntry::getInstance()->initXrSwapchains(); - }); + }) } else { xr::XrEntry::getInstance()->initXrSwapchains(); } @@ -551,8 +608,9 @@ void XRInterface::createXRSwapchains() { const std::vector &XRInterface::getXRSwapchains() { #if CC_USE_XR CC_LOG_INFO("[XR] getXRSwapchains"); - if (_xrSwapchains.size() == 0) + if (_xrSwapchains.empty()) { _xrSwapchains = xr::XrEntry::getInstance()->getCocosXrSwapchains(); + } #endif return _xrSwapchains; } @@ -563,9 +621,13 @@ gfx::Format XRInterface::getXRSwapchainFormat() { #if CC_USE_GLES3 if (swapchainFormat == GL_SRGB_ALPHA_EXT) { return gfx::Format::SRGB8_A8; - } else if (swapchainFormat == GL_RGBA8) { + } + + if (swapchainFormat == GL_RGBA8) { return gfx::Format::RGBA8; - } else if (swapchainFormat == GL_BGRA8_EXT) { + } + + if (swapchainFormat == GL_BGRA8_EXT) { return gfx::Format::BGRA8; } #endif @@ -573,9 +635,13 @@ gfx::Format XRInterface::getXRSwapchainFormat() { #if CC_USE_VULKAN if (swapchainFormat == VK_FORMAT_R8G8B8A8_SRGB) { return gfx::Format::SRGB8_A8; - } else if (swapchainFormat == VK_FORMAT_R8G8B8A8_UNORM) { + } + + if (swapchainFormat == VK_FORMAT_R8G8B8A8_UNORM) { return gfx::Format::RGBA8; - } else if (swapchainFormat == VK_FORMAT_B8G8R8A8_UNORM) { + } + + if (swapchainFormat == VK_FORMAT_B8G8R8A8_UNORM) { return gfx::Format::BGRA8; } #endif @@ -684,6 +750,11 @@ EGLSurfaceType XRInterface::acquireEGLSurfaceType(uint32_t typedID) { bool XRInterface::platformLoopStart() { #if CC_USE_XR // CC_LOG_INFO("[XR] platformLoopStart"); + #if CC_USE_XR_REMOTE_PREVIEW + if (_xrRemotePreviewManager && !_xrRemotePreviewManager->isStarted()) { + _xrRemotePreviewManager->start(); + } + #endif return xr::XrEntry::getInstance()->platformLoopStart(); #else return false; @@ -707,11 +778,21 @@ bool XRInterface::beginRenderFrame() { { if (IS_ENABLE_XR_LOG) CC_LOG_INFO("[XR] [RT] beginRenderFrame.%lld", frameId); xr::XrEntry::getInstance()->frameStart(); - }); + }) + #if CC_USE_XR_REMOTE_PREVIEW + if (_xrRemotePreviewManager) { + _xrRemotePreviewManager->tick(); + } + #endif return true; - } else { - return xr::XrEntry::getInstance()->frameStart(); } + + #if CC_USE_XR_REMOTE_PREVIEW + if (_xrRemotePreviewManager) { + _xrRemotePreviewManager->tick(); + } + #endif + return xr::XrEntry::getInstance()->frameStart(); #else return false; #endif @@ -734,9 +815,9 @@ bool XRInterface::beginRenderEyeFrame(uint32_t eye) { { if (IS_ENABLE_XR_LOG) CC_LOG_INFO("[XR] [RT] beginRenderEyeFrame %d", eye); xr::XrEntry::getInstance()->renderLoopStart(eye); - }); + }) } else { - xr::XrEntry::getInstance()->renderLoopStart(eye); + xr::XrEntry::getInstance()->renderLoopStart(static_cast(eye)); } #else CC_UNUSED_PARAM(eye); @@ -753,9 +834,9 @@ bool XRInterface::endRenderEyeFrame(uint32_t eye) { { if (IS_ENABLE_XR_LOG) CC_LOG_INFO("[XR] [RT] endRenderEyeFrame %d", eye); xr::XrEntry::getInstance()->renderLoopEnd(eye); - }); + }) } else { - xr::XrEntry::getInstance()->renderLoopEnd(eye); + xr::XrEntry::getInstance()->renderLoopEnd(static_cast(eye)); } #else CC_UNUSED_PARAM(eye); @@ -765,8 +846,10 @@ bool XRInterface::endRenderEyeFrame(uint32_t eye) { bool XRInterface::endRenderFrame() { #if CC_USE_XR - if (IS_ENABLE_XR_LOG) CC_LOG_INFO("[XR] endRenderFrame.%d", - cc::ApplicationManager::getInstance()->getCurrentAppSafe()->getEngine()->getTotalFrames()); + if (IS_ENABLE_XR_LOG) { + CC_LOG_INFO("[XR] endRenderFrame.%d", + cc::ApplicationManager::getInstance()->getCurrentAppSafe()->getEngine()->getTotalFrames()); + } if (gfx::DeviceAgent::getInstance()) { ENQUEUE_MESSAGE_0(gfx::DeviceAgent::getInstance()->getMessageQueue(), @@ -776,11 +859,16 @@ bool XRInterface::endRenderFrame() { gfx::DeviceAgent::getInstance()->presentSignal(); if (IS_ENABLE_XR_LOG) CC_LOG_INFO("[XR] [RT] presentSignal endRenderFrame.%d", cc::ApplicationManager::getInstance()->getCurrentAppSafe()->getEngine()->getTotalFrames()); - }); + }) _committedFrame = true; // CC_LOG_INFO("[XR] endRenderFrame pass presentWait errno %d", errno); } else { xr::XrEntry::getInstance()->frameEnd(); + #if CC_USE_XR_REMOTE_PREVIEW + if (_xrRemotePreviewManager) { + _xrRemotePreviewManager->tick(); + } + #endif } #endif return true; @@ -807,6 +895,17 @@ ccstd::vector XRInterface::getHMDViewPosition(uint32_t eye, int trackingT #endif } +ccstd::vector XRInterface::getXREyeFov(uint32_t eye) { +#if CC_USE_XR + return xr::XrEntry::getInstance()->getEyeFov(eye); +#else + CC_UNUSED_PARAM(eye); + ccstd::vector res; + res.reserve(4); + return res; +#endif +} + ccstd::vector XRInterface::getXRViewProjectionData(uint32_t eye, float near, float far) { #if CC_USE_XR return xr::XrEntry::getInstance()->computeViewProjection(eye, near, far, 1.0F); @@ -838,4 +937,57 @@ void XRInterface::bindXREyeWithRenderWindow(void *window, xr::XREye eye) { _xrWindowMap.emplace(std::make_pair(window, eye)); } } + +void XRInterface::loadAssetsImage(const std::string &imageInfo) { + // name|@assets/TrackingImage_SpacesTown.png|0.18|0.26 + ccstd::vector segments = StringUtil::split(imageInfo, "|"); + std::string imageName = segments.at(0); + std::string imagePath = segments.at(1); + auto physicalSizeX = static_cast(atof(segments.at(2).c_str())); + auto physicalSizeY = static_cast(atof(segments.at(3).c_str())); + auto *spaceTownImage = new Image(); + spaceTownImage->addRef(); + bool res = spaceTownImage->initWithImageFile(imagePath); + if (!res) { + CC_LOG_ERROR("[XRInterface] loadAssetsImage init failed, %s!!!", imageInfo.c_str()); + return; + } + uint32_t imageWidth = spaceTownImage->getWidth(); + uint32_t imageHeight = spaceTownImage->getHeight(); + const uint32_t bufferSize = imageWidth * imageHeight * 3; + auto *buffer = new uint8_t[bufferSize]; + for (unsigned int j = 0; j < imageHeight; ++j) { + for (unsigned int i = 0; i < imageWidth; ++i) { + const unsigned int pixel = i + j * imageWidth; + const unsigned int pixelFlip = i + (imageHeight - j - 1) * imageWidth; + + const uint8_t *originalPixel = &spaceTownImage->getData()[static_cast(pixel * 4)]; + uint8_t *convertedPixel = &buffer[static_cast(pixelFlip * 3)]; + convertedPixel[0] = originalPixel[0]; + convertedPixel[1] = originalPixel[1]; + convertedPixel[2] = originalPixel[2]; + } + } + spaceTownImage->release(); + + auto app = CC_CURRENT_APPLICATION(); + if (!app) { + CC_LOG_ERROR("[XRInterface] loadAssetsImage callback failed, application not exist!!!"); + return; + } + auto engine = app->getEngine(); + CC_ASSERT(engine != nullptr); + engine->getScheduler()->performFunctionInCocosThread([=]() { + xr::XRTrackingImageData candidateImage; + candidateImage.friendlyName = imageName; + candidateImage.bufferSize = bufferSize; + candidateImage.buffer = buffer; + candidateImage.pixelSizeWidth = imageWidth; + candidateImage.pixelSizeHeight = imageHeight; + candidateImage.physicalWidth = physicalSizeX; + candidateImage.physicalHeight = physicalSizeY; + setXRConfig(xr::XRConfigKey::IMAGE_TRACKING_CANDIDATEIMAGE, static_cast(&candidateImage)); + }); +} + } // namespace cc diff --git a/native/cocos/platform/java/modules/XRInterface.h b/native/cocos/platform/java/modules/XRInterface.h index a2b19db8b0..9b59f399d6 100644 --- a/native/cocos/platform/java/modules/XRInterface.h +++ b/native/cocos/platform/java/modules/XRInterface.h @@ -25,7 +25,16 @@ #pragma once +#include "base/Ptr.h" +#include "base/ThreadPool.h" #include "platform/interfaces/modules/IXRInterface.h" +#if CC_USE_XR_REMOTE_PREVIEW +#include "xr/XRRemotePreviewManager.h" +#endif +namespace se { +class Object; +} + namespace cc { class XRInterface : public IXRInterface { @@ -86,11 +95,19 @@ public: ccstd::vector getHMDViewPosition(uint32_t eye, int trackingType) override; ccstd::vector getXRViewProjectionData(uint32_t eye, float near, float far) override; + ccstd::vector getXREyeFov(uint32_t eye) override; // renderwindow xr::XREye getXREyeByRenderWindow(void *window) override; void bindXREyeWithRenderWindow(void *window, xr::XREye eye) override; private: + void loadAssetsImage(const std::string &imageInfo); + void dispatchGamepadEventInternal(const xr::XRControllerEvent &xrControllerEvent); + void dispatchHandleEventInternal(const xr::XRControllerEvent &xrControllerEvent); + void dispatchHMDEventInternal(const xr::XRControllerEvent &xrControllerEvent); + ControllerEvent _controllerEvent; + se::Object *_jsPoseEventArray{nullptr}; + #if CC_USE_VULKAN PFN_vkGetInstanceProcAddr _vkGetInstanceProcAddr{nullptr}; VkPhysicalDevice _vkPhysicalDevice{nullptr}; @@ -109,6 +126,10 @@ private: std::unordered_map _xrWindowMap; std::unordered_map _eglSurfaceTypeMap; bool _committedFrame{false}; +#if CC_USE_XR_REMOTE_PREVIEW + cc::IntrusivePtr _xrRemotePreviewManager{nullptr}; +#endif + LegacyThreadPool *_gThreadPool{nullptr}; }; } // namespace cc diff --git a/native/cocos/xr/XRRemotePreviewManager.cpp b/native/cocos/xr/XRRemotePreviewManager.cpp new file mode 100644 index 0000000000..4646871cc8 --- /dev/null +++ b/native/cocos/xr/XRRemotePreviewManager.cpp @@ -0,0 +1,375 @@ +/**************************************************************************** + Copyright (c) 2021-2022 Xiamen Yaji Software Co., Ltd. + + http://www.cocos.com + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated engine source code (the "Software"), a limited, + worldwide, royalty-free, non-assignable, revocable and non-exclusive license + to use Cocos Creator solely to develop games on your target platforms. You shall + not use Cocos Creator software for developing other software or tools that's + used for developing games. You are not granted to publish, distribute, + sublicense, and/or sell copies of Cocos Creator. + + The software or tools in this License Agreement are licensed, not sold. + Xiamen Yaji Software Co., Ltd. reserves all rights not expressly granted to you. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +****************************************************************************/ + +#include "base/Log.h" +#include "base/Macros.h" +#include "base/std/container/vector.h" +#include "base/std/container/list.h" +#include "base/std/container/string.h" +#include "base/std/container/unordered_map.h" +#include "application/ApplicationManager.h" +#include "platform/java/modules/XRInterface.h" +#include "math/Quaternion.h" + +#if CC_USE_XR +#include "Xr.h" +#if ANDROID +#include +#endif +#include "XRRemotePreviewManager.h" +#if CC_USE_WEBSOCKET_SERVER +#include "network/WebSocketServer.h" +#endif + +namespace cc { +#define SOCKET_PORT 8989 +// 11259375 +const int32_t PACKET_HEAD_CODE = 0x000ABCDEF; + +XRRemotePreviewManager::XRRemotePreviewManager() { + +} + +XRRemotePreviewManager::~XRRemotePreviewManager() { + +} + +#if CC_USE_WEBSOCKET_SERVER +void XRRemotePreviewManager::onClientConnected(const std::shared_ptr &connection) { + CC_LOG_INFO("[XRRemotePreviewManager] onClientConnected.%p", connection.get()); + XRDeviceInfo deviceInfo; + deviceInfo.type = static_cast(XRDataPackageType::DPT_MSG_DEVICE_INFO); + memset(deviceInfo.deviceName, 0, sizeof(char) * 32); + strcpy(deviceInfo.deviceName, "XRDevice"); +#if ANDROID + char man[PROP_VALUE_MAX + 1]; + char mod[PROP_VALUE_MAX + 1]; + /* A length 0 value indicates that the property is not defined */ + int lman = __system_property_get("ro.product.manufacturer", man); + int lmod = __system_property_get("ro.product.model", mod); + int len = lman + lmod; + char *pname = nullptr; + if (len > 0) { + pname = static_cast(malloc(len + 2)); + snprintf(pname, len + 2, "%s %s", lman > 0 ? man : "", lmod > 0 ? mod : ""); + strcpy(deviceInfo.deviceName, pname); + } + if (pname) free(pname); +#endif + _wssConnections = _webSocketServer->getConnections(); + _xr = CC_GET_XR_INTERFACE(); + if (_xr) { + ccstd::vector leftEyeFov = _xr->getXREyeFov(static_cast(cc::xr::XREye::LEFT)); + ccstd::vector rightEyeFov = _xr->getXREyeFov(static_cast(cc::xr::XREye::RIGHT)); + for (uint32_t i = 0; i < 4; i++) { + CC_LOG_INFO("[XRRemotePreviewManager] eye fov.%d, L.%f, R.%f.", i, leftEyeFov[i], rightEyeFov[i]); + deviceInfo.leftEyeFov[i] = leftEyeFov[i] * math::RAD_TO_DEG; + deviceInfo.rightEyeFov[i] = rightEyeFov[i] * math::RAD_TO_DEG; + } + deviceInfo.version = static_cast(_xr->getRuntimeVersion()); + deviceInfo.xrVendorType = static_cast(_xr->getVendor()); + sendDeviceInfo(deviceInfo); + } +} + +void XRRemotePreviewManager::onClientDisconnected(int closeCode, const ccstd::string &closeReason) { + CC_LOG_INFO("[XRRemotePreviewManager] onClientDisconnected.%d|%s", closeCode, closeReason.c_str()); + this->_isConnectionChanged = true; +} + +void XRRemotePreviewManager::onConnection(const std::shared_ptr &connection) { + CC_LOG_INFO("[XRRemotePreviewManager] onConnection"); + + auto map = connection->getHeaders(); + auto mapBegin = map.begin(); + while (mapBegin != map.end()) { + CC_LOG_INFO("[XRRemotePreviewManager] connection Headers.%s=%s", + (*mapBegin).first.c_str(), + (*mapBegin).second.c_str()); + mapBegin++; + } + + auto protocolsList = connection->getProtocols(); + auto listBegin = protocolsList.begin(); + while (listBegin != protocolsList.end()) { + CC_LOG_INFO("[XRRemotePreviewManager] connection Protocols.%s", (*listBegin).c_str()); + listBegin++; + } + + onClientConnected(connection); + + connection->setOnClose([=](int closeCode, const ccstd::string &closeReason) { + onClientDisconnected(closeCode, closeReason); + }); +} +#endif + +void XRRemotePreviewManager::start() { +#if CC_USE_WEBSOCKET_SERVER + _webSocketServer = std::make_shared(); + + _webSocketServer->setOnBegin([]() { + CC_LOG_INFO("[XRRemotePreviewManager] onBegin"); + }); + + _webSocketServer->setOnEnd([]() { + CC_LOG_INFO("[XRRemotePreviewManager] onEnd"); + }); + + _webSocketServer->setOnClose([](const ccstd::string &msg) { + CC_LOG_INFO("[XRRemotePreviewManager] onClose.%s", msg.c_str()); + }); + + _webSocketServer->setOnListening([](const ccstd::string &msg) { + CC_LOG_INFO("[XRRemotePreviewManager] onListening.%s", msg.c_str()); + }); + + _webSocketServer->setOnError([](const ccstd::string &msg) { + CC_LOG_INFO("[XRRemotePreviewManager] onError.%s", msg.c_str()); + }); + + _webSocketServer->setOnConnection(std::bind(&XRRemotePreviewManager::onConnection, this, std::placeholders::_1)); + + WebSocketServer::listenAsync(_webSocketServer, SOCKET_PORT, "127.0.0.1", nullptr); + CC_LOG_INFO("[XRRemotePreviewManager] listenAsync ready!"); +#endif + _isStarted = true; +} + +void XRRemotePreviewManager::packDeviceInfoData(const XRDeviceInfo &info, char *data) { + memcpy(data, &PACKET_HEAD_CODE, sizeof(char) * 4); + size_t infoSize = sizeof(info); + memcpy(data + 4, &infoSize, sizeof(char) * 4); + memcpy(data + 8, &info, infoSize); +} + +void XRRemotePreviewManager::packDevicePoseData(const XRPoseInfo &info, char *data) { + memcpy(data, &PACKET_HEAD_CODE, sizeof(char) * 4); + size_t infoSize = sizeof(info); + memcpy(data + 4, &infoSize, sizeof(char) * 4); + memcpy(data + 8, &info, infoSize); +} + +void XRRemotePreviewManager::packControllerKeyData(const XRControllerKeyInfo &info, char *data) { + memcpy(data, &PACKET_HEAD_CODE, sizeof(char) * 4); + size_t infoSize = sizeof(info); + memcpy(data + 4, &infoSize, sizeof(char) * 4); + memcpy(data + 8, &info, infoSize); +} + +void XRRemotePreviewManager::packCommonMessageData(const XRCommonMessage &info, char *data) { + memcpy(data, &PACKET_HEAD_CODE, sizeof(char) * 4); + size_t infoSize = sizeof(info); + memcpy(data + 4, &infoSize, sizeof(char) * 4); + memcpy(data + 8, &info, infoSize); +} + +void XRRemotePreviewManager::sendMessage(const ccstd::string &message) { +#if CC_USE_WEBSOCKET_SERVER + if (_webSocketServer && _isConnectionChanged) { + _isConnectionChanged = false; + _wssConnections = _webSocketServer->getConnections(); + } + + if (_webSocketServer && !_wssConnections.empty()) { + for (auto &wssConn : _wssConnections) { + if (wssConn->getReadyState() == WebSocketServerConnection::ReadyState::OPEN) { + XRCommonMessage xrCommonMsg; + xrCommonMsg.type = static_cast(XRDataPackageType::DPT_MSG_STRING); + xrCommonMsg.message = message; + size_t dataLen = 8 + sizeof(xrCommonMsg); + char data[dataLen]; + packCommonMessageData(xrCommonMsg, data); + wssConn->sendBinaryAsync(data, dataLen, nullptr); + } else { + CC_LOG_INFO("[XRRemotePreviewManager] sendMessage [%s] failed !!!", message.c_str()); + } + } + } else { + CC_LOG_INFO("[XRRemotePreviewManager] sendMessage [%s] failed [wss invalid] !!!", message.c_str()); + } +#endif +} + +void XRRemotePreviewManager::sendDeviceInfo(const XRDeviceInfo &info) { +#if CC_USE_WEBSOCKET_SERVER + if (_webSocketServer && _isConnectionChanged) { + _isConnectionChanged = false; + _wssConnections = _webSocketServer->getConnections(); + } + + if (_webSocketServer && !_wssConnections.empty()) { + for (auto &wssConn: _wssConnections) { + if (wssConn->getReadyState() == WebSocketServerConnection::ReadyState::OPEN) { + size_t dataLen = 8 + sizeof(info); + char data[dataLen]; + packDeviceInfoData(info, data); + wssConn->sendBinaryAsync(data, dataLen, nullptr); + } else { + CC_LOG_INFO("[XRRemotePreviewManager] sendDeviceInfo failed !!!"); + } + } + } else { + CC_LOG_ERROR("[XRRemotePreviewManager] sendDeviceInfo failed [wss invalid] !!!"); + } +#endif +} + +void XRRemotePreviewManager::sendPoseInfo(const XRPoseInfo &info) { +#if CC_USE_WEBSOCKET_SERVER + if (_webSocketServer && _isConnectionChanged) { + _isConnectionChanged = false; + _wssConnections = _webSocketServer->getConnections(); + } + + if (_webSocketServer && !_wssConnections.empty()) { + for (auto &wssConn: _wssConnections) { + if (wssConn->getReadyState() == WebSocketServerConnection::ReadyState::OPEN) { + size_t dataLen = 8 + sizeof(info); + char data[dataLen]; + packDevicePoseData(info, data); + wssConn->sendBinaryAsync(data, dataLen, nullptr); + } else { + CC_LOG_ERROR("[XRRemotePreviewManager] sendPoseInfo failed %d!!!", wssConn->getReadyState()); + } + } + } +#endif +} + +void XRRemotePreviewManager::sendControllerKeyInfo(const ControllerInfo::ButtonInfo &info) { + #if CC_USE_WEBSOCKET_SERVER + if (_webSocketServer && _isConnectionChanged) { + _isConnectionChanged = false; + _wssConnections = _webSocketServer->getConnections(); + } + + if (_webSocketServer && !_wssConnections.empty()) { + for (auto &wssConn : _wssConnections) { + if (wssConn->getReadyState() == WebSocketServerConnection::ReadyState::OPEN) { + XRControllerKeyInfo ctrlKeyInfo; + ctrlKeyInfo.type = static_cast(XRDataPackageType::DPT_MSG_CONTROLLER_KEY); + ctrlKeyInfo.isButtonPressed = info.isPress; + ctrlKeyInfo.stickKeyCode = static_cast(info.key); + ctrlKeyInfo.keyEventType = static_cast(XRKeyEventType::KET_CLICK); + size_t dataLen = 8 + sizeof(ctrlKeyInfo); + char data[dataLen]; + packControllerKeyData(ctrlKeyInfo, data); + wssConn->sendBinaryAsync(data, dataLen, nullptr); + } else { + CC_LOG_ERROR("[XRRemotePreviewManager] sendControllerKeyInfo button failed !!!"); + } + } + } + #endif +} + +void XRRemotePreviewManager::sendControllerKeyInfo(const ControllerInfo::AxisInfo &info) { + #if CC_USE_WEBSOCKET_SERVER + if (_webSocketServer && _isConnectionChanged) { + _isConnectionChanged = false; + _wssConnections = _webSocketServer->getConnections(); + } + + if (_webSocketServer && !_wssConnections.empty()) { + for (auto &wssConn : _wssConnections) { + if (wssConn->getReadyState() == WebSocketServerConnection::ReadyState::OPEN) { + XRControllerKeyInfo ctrlKeyInfo; + ctrlKeyInfo.type = static_cast(XRDataPackageType::DPT_MSG_CONTROLLER_KEY); + ctrlKeyInfo.stickAxisCode = static_cast(info.axis); + ctrlKeyInfo.stickAxisValue = info.value; + if (info.axis == StickAxisCode::LEFT_STICK_X || info.axis == StickAxisCode::LEFT_STICK_Y || + info.axis == StickAxisCode::RIGHT_STICK_X || info.axis == StickAxisCode::RIGHT_STICK_Y) { + ctrlKeyInfo.keyEventType = static_cast(XRKeyEventType::KET_STICK); + } else { + ctrlKeyInfo.keyEventType = static_cast(XRKeyEventType::KET_GRAB); + } + size_t dataLen = 8 + sizeof(ctrlKeyInfo); + char data[dataLen]; + packControllerKeyData(ctrlKeyInfo, data); + wssConn->sendBinaryAsync(data, dataLen, nullptr); + } else { + CC_LOG_ERROR("[XRRemotePreviewManager] sendControllerKeyInfo axis failed !!!"); + } + } + } + #endif +} + +void XRRemotePreviewManager::resume() { + CC_LOG_INFO("[XRRemotePreviewManager] resume"); + sendMessage("resume"); +} + +void XRRemotePreviewManager::pause() { + CC_LOG_INFO("[XRRemotePreviewManager] pause"); + sendMessage("pause"); +} + +void XRRemotePreviewManager::stop() { + _isStarted = false; + CC_LOG_INFO("[XRRemotePreviewManager] stop"); + _webSocketServer->closeAsync(); + _webSocketServer = nullptr; +} + +bool XRRemotePreviewManager::isStarted() const { + return _isStarted; +} + +void XRRemotePreviewManager::tick() { + if (this->_isStarted) { + xr::XrEntry::getInstance()->getXrPose(_leftEyePose, _rightEyePose, _leftControllerPose, _rightControllerPose); + _devicePoseInfo.type = static_cast (XRDataPackageType::DPT_MSG_POSE_DATA); + _devicePoseInfo.hmdOrientation[0] = _leftEyePose.qx; + _devicePoseInfo.hmdOrientation[1] = _leftEyePose.qy; + _devicePoseInfo.hmdOrientation[2] = _leftEyePose.qz; + _devicePoseInfo.hmdOrientation[3] = _leftEyePose.qw; + _devicePoseInfo.hmdPosition[0] = (_leftEyePose.px + _rightEyePose.px) * 0.5F; + _devicePoseInfo.hmdPosition[1] = (_leftEyePose.py + _rightEyePose.py) * 0.5F; + _devicePoseInfo.hmdPosition[2] = (_leftEyePose.pz + _rightEyePose.pz) * 0.5F; + + _devicePoseInfo.leftControllerOrientation[0] = _leftControllerPose.qx; + _devicePoseInfo.leftControllerOrientation[1] = _leftControllerPose.qy; + _devicePoseInfo.leftControllerOrientation[2] = _leftControllerPose.qz; + _devicePoseInfo.leftControllerOrientation[3] = _leftControllerPose.qw; + _devicePoseInfo.leftControllerPosition[0] = _leftControllerPose.px; + _devicePoseInfo.leftControllerPosition[1] = _leftControllerPose.py; + _devicePoseInfo.leftControllerPosition[2] = _leftControllerPose.pz; + + _devicePoseInfo.rightControllerOrientation[0] = _rightControllerPose.qx; + _devicePoseInfo.rightControllerOrientation[1] = _rightControllerPose.qy; + _devicePoseInfo.rightControllerOrientation[2] = _rightControllerPose.qz; + _devicePoseInfo.rightControllerOrientation[3] = _rightControllerPose.qw; + _devicePoseInfo.rightControllerPosition[0] = _rightControllerPose.px; + _devicePoseInfo.rightControllerPosition[1] = _rightControllerPose.py; + _devicePoseInfo.rightControllerPosition[2] = _rightControllerPose.pz; + this->sendPoseInfo(_devicePoseInfo); + } +} + +} // namespace cc + +#endif diff --git a/native/cocos/xr/XRRemotePreviewManager.h b/native/cocos/xr/XRRemotePreviewManager.h new file mode 100644 index 0000000000..85235d8738 --- /dev/null +++ b/native/cocos/xr/XRRemotePreviewManager.h @@ -0,0 +1,135 @@ +/**************************************************************************** + Copyright (c) 2021-2022 Xiamen Yaji Software Co., Ltd. + + http://www.cocos.com + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated engine source code (the "Software"), a limited, + worldwide, royalty-free, non-assignable, revocable and non-exclusive license + to use Cocos Creator solely to develop games on your target platforms. You shall + not use Cocos Creator software for developing other software or tools that's + used for developing games. You are not granted to publish, distribute, + sublicense, and/or sell copies of Cocos Creator. + + The software or tools in this License Agreement are licensed, not sold. + Xiamen Yaji Software Co., Ltd. reserves all rights not expressly granted to you. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +****************************************************************************/ + +#pragma once +#include +#include +#include "base/RefCounted.h" +#include "XRCommon.h" + +namespace cc { +class IXRInterface; +namespace network { +class WebSocketServer; +class WebSocketServerConnection; +} +using namespace network; + +#if CC_USE_XR +enum class XRDataPackageType { + DPT_MSG_DEVICE_INFO = 1, + DPT_MSG_POSE_DATA, + DPT_MSG_CONTROLLER_KEY, + DPT_MSG_STRING +}; + +enum class XRKeyEventType { + KET_CLICK, + KET_STICK, + KET_GRAB +}; + +#pragma pack(1) +struct XRCommonMessage { + int16_t type; + ccstd::string message; +}; +#pragma pack () + +#pragma pack(1) +struct XRDeviceInfo { + int16_t type; + int16_t version; + char deviceName[32]; + int16_t xrVendorType; + float leftEyeFov[4]; //L-R-D-U + float rightEyeFov[4]; +}; +#pragma pack () + +#pragma pack(1) +struct XRPoseInfo { + int16_t type; + float hmdOrientation[4]; + float hmdPosition[3]; + float leftControllerOrientation[4]; + float leftControllerPosition[3]; + float rightControllerOrientation[4]; + float rightControllerPosition[3]; +}; +#pragma pack () + +#pragma pack(1) +struct XRControllerKeyInfo { + int16_t type; + int16_t keyEventType; + int16_t stickAxisCode{0}; + float stickAxisValue{0}; + int16_t stickKeyCode{0}; + bool isButtonPressed{false}; +}; +#pragma pack () + +class XRRemotePreviewManager : public RefCounted { + public: + XRRemotePreviewManager(); + ~XRRemotePreviewManager() override; + void start(); + void sendDeviceInfo(const XRDeviceInfo &info); + void sendPoseInfo(const XRPoseInfo &info); + void sendControllerKeyInfo(const ControllerInfo::ButtonInfo &info); + void sendControllerKeyInfo(const ControllerInfo::AxisInfo &info); + void tick(); + void resume(); + void pause(); + void stop(); + bool isStarted() const; + private: + void sendMessage(const ccstd::string &message); + static void packCommonMessageData(const XRCommonMessage &message, char *data); + static void packDeviceInfoData(const XRDeviceInfo &info, char *data); + static void packDevicePoseData(const XRPoseInfo &info, char *data); + static void packControllerKeyData(const XRControllerKeyInfo &info, char *data); +#if CC_USE_WEBSOCKET_SERVER + void onConnection(const std::shared_ptr &connection); + void onClientConnected(const std::shared_ptr &connection); + void onClientDisconnected(int closeCode, const ccstd::string &closeReason); + std::shared_ptr _webSocketServer{nullptr}; + ccstd::vector> _wssConnections; +#endif + bool _isStarted{false}; + bool _isConnectionChanged{false}; + IXRInterface *_xr{nullptr}; + cc::xr::XRPose _leftEyePose{cc::xr::XRPose::Type::VIEW_LEFT, true}; + cc::xr::XRPose _rightEyePose{cc::xr::XRPose::Type::VIEW_RIGHT, true}; + cc::xr::XRPose _leftControllerPose{cc::xr::XRPose::Type::HAND_LEFT, true}; + cc::xr::XRPose _rightControllerPose{cc::xr::XRPose::Type::HAND_RIGHT, true}; + XRPoseInfo _devicePoseInfo; + private: + CC_DISALLOW_COPY_MOVE_ASSIGN(XRRemotePreviewManager); +}; +#endif + +} // namespace cc diff --git a/native/cocos/xr/Xr.h b/native/cocos/xr/Xr.h index d85bc12d18..3b67b4d4cb 100644 --- a/native/cocos/xr/Xr.h +++ b/native/cocos/xr/Xr.h @@ -30,6 +30,10 @@ #include #include "XRCommon.h" +#ifndef XR_USE_GRAPHICS_API_OPENGL_ES +#define XR_USE_GRAPHICS_API_OPENGL_ES 1 +#endif + #ifdef XR_USE_GRAPHICS_API_VULKAN #include "vulkan/vulkan_core.h" #endif @@ -75,6 +79,8 @@ public: #ifdef XR_USE_GRAPHICS_API_OPENGL_ES virtual void initXrSession(PFNGLES3WLOADPROC gles3wLoadFuncProc, void *eglDisplay, void *eglConfig, void *eglDefaultContext) = 0; + virtual void setXrFrameBuffer(unsigned int fbo) = 0; + virtual unsigned int getXrFrameBuffer() = 0; virtual void attachXrFramebufferTexture2D() = 0; @@ -108,6 +114,8 @@ public: virtual std::vector computeViewProjection(uint32_t index, float nearZ, float farZ, float scaleF) = 0; + virtual std::vector getEyeFov(uint32_t eye) = 0; + virtual uint32_t getSwapchainImageIndex() = 0; virtual void setMultisamplesRTT(int num) = 0; @@ -149,6 +157,11 @@ public: virtual std::string getXRStringConfig(int key) = 0; virtual void *getXRPointerConfig(int key) = 0; + + virtual void getXrPose(cc::xr::XRPose &leftEyePose, + cc::xr::XRPose &rightEyePose, + cc::xr::XRPose &leftControllerPose, + cc::xr::XRPose &rightControllerPose) = 0; }; } // namespace xr