From 52484a619ce2e3cd8740776aa6e563054a334bb0 Mon Sep 17 00:00:00 2001 From: oahc09 Date: Fri, 24 Feb 2023 18:07:47 +0800 Subject: [PATCH] refactor:xr 2d orthographic camera ui display effect (#14371) --- .../interfaces/modules/IXRInterface.h | 13 ++++ .../platform/java/modules/XRInterface.cpp | 76 +++++++++++++++++++ .../cocos/platform/java/modules/XRInterface.h | 1 + native/cocos/scene/Camera.cpp | 11 +-- 4 files changed, 92 insertions(+), 9 deletions(-) diff --git a/native/cocos/platform/interfaces/modules/IXRInterface.h b/native/cocos/platform/interfaces/modules/IXRInterface.h index 5abf603ccf..42933cac6c 100644 --- a/native/cocos/platform/interfaces/modules/IXRInterface.h +++ b/native/cocos/platform/interfaces/modules/IXRInterface.h @@ -38,6 +38,9 @@ namespace cc { namespace gfx { class GLES3GPUContext; } +namespace scene { +class Camera; +} enum class EGLSurfaceType { NONE, @@ -323,5 +326,15 @@ public: * @param appCmd */ virtual void handleAppCommand(int appCmd) = 0; + + /** + * @en adapt orthographic matrix(projection and view) + * @zh 适配正交相机 + * @param camera + * @param preTransform + * @param proj + * @param view + */ + virtual void adaptOrthographicMatrix(cc::scene::Camera *camera, const ccstd::array &preTransform, Mat4 &proj, Mat4 &view) = 0; }; } // namespace cc diff --git a/native/cocos/platform/java/modules/XRInterface.cpp b/native/cocos/platform/java/modules/XRInterface.cpp index d803c1aa22..8ae1e99fa7 100644 --- a/native/cocos/platform/java/modules/XRInterface.cpp +++ b/native/cocos/platform/java/modules/XRInterface.cpp @@ -35,6 +35,11 @@ #include "cocos/bindings/jswrapper/SeApi.h" #include "java/jni/JniHelper.h" #include "renderer/GFXDeviceManager.h" +#include "platform/interfaces/modules/ISystemWindow.h" +#include "platform/interfaces/modules/ISystemWindowManager.h" +#include "scene/Camera.h" +#include "scene/RenderWindow.h" +#include "core/scene-graph/Node.h" #ifdef CC_USE_VULKAN #include "gfx-vulkan/VKDevice.h" #endif @@ -1016,6 +1021,77 @@ void XRInterface::handleAppCommand(int appCmd) { CC_UNUSED_PARAM(appCmd); #endif } + +void XRInterface::adaptOrthographicMatrix(cc::scene::Camera *camera, const ccstd::array &preTransform, Mat4 &proj, Mat4 &view) { +#if CC_USE_XR + const float orthoHeight = camera->getOrthoHeight(); + const float near = camera->getNearClip(); + const float far = camera->getFarClip(); + const float aspect = camera->getAspect(); + auto *swapchain = camera->getWindow()->getSwapchain(); + const auto &orientation = swapchain ? swapchain->getSurfaceTransform() : gfx::SurfaceTransform::IDENTITY; + cc::xr::XREye wndXREye = getXREyeByRenderWindow(camera->getWindow()); + const auto &xrFov = getXREyeFov(static_cast(wndXREye)); + const float ipdValue = getXRConfig(xr::XRConfigKey::DEVICE_IPD).getFloat(); + const float projectionSignY = gfx::Device::getInstance()->getCapabilities().clipSpaceSignY; + Mat4 orthoProj; + auto *systemWindow = CC_GET_MAIN_SYSTEM_WINDOW(); + const float deviceAspect = systemWindow ? (systemWindow->getViewSize().width / systemWindow->getViewSize().height) : 1.777F; + // device's fov may be asymmetry + float radioLeft = 1.0F; + float radioRight = 1.0F; + if (wndXREye == xr::XREye::LEFT) { + radioLeft = fabs(tanf(xrFov[0])) / fabs(tanf(xrFov[1])); + } else if (wndXREye == xr::XREye::RIGHT) { + radioRight = fabs(tanf(xrFov[1])) / fabs(tanf(xrFov[0])); + } + + const float x = orthoHeight * deviceAspect; + const float y = orthoHeight; + Mat4::createOrthographicOffCenter(-x * radioLeft, x * radioRight, -y, y, near, far, + gfx::Device::getInstance()->getCapabilities().clipSpaceMinZ, projectionSignY, + static_cast(orientation), &orthoProj); + // keep scale to [-1, 1] only use offset + orthoProj.m[0] = preTransform[0] / x; + orthoProj.m[5] = preTransform[3] * projectionSignY / y; + + Mat4 eyeCameraProj; + const auto &projFloat = getXRViewProjectionData(static_cast(wndXREye), 0.001F, 1000.0F); + std::memcpy(eyeCameraProj.m, projFloat.data(), sizeof(float) * 16); + Vec4 point{ipdValue * 0.5F, 0.0F, -1.0F, 1.0F}; + eyeCameraProj.transformVector(&point); + + Mat4 objectTranslateMat; + // point in [-1,1] convert to [-aspect,aspect] + if (wndXREye == xr::XREye::LEFT) { + objectTranslateMat.translate(point.x * orthoHeight * 0.5F * aspect, 0.0F, 0.0F); + } else if (wndXREye == xr::XREye::RIGHT) { + objectTranslateMat.translate(-point.x * orthoHeight * 0.5F * aspect, 0.0F, 0.0F); + } + // update view matrix + view = camera->getNode()->getWorldMatrix().getInversed(); + Mat4 scaleMat{}; + scaleMat.scale(camera->getNode()->getWorldScale()); + // remove scale + Mat4::multiply(scaleMat, view, &view); + Mat4::multiply(objectTranslateMat, view, &view); + + // fit width, scale deviceAspect to aspect + const float ndcXScale = getXRConfig(xr::XRConfigKey::SPLIT_AR_GLASSES).getBool() ? (1.0F + point.x * 0.5F) : 1.0F; + const float ndcYScale = aspect / deviceAspect; + Mat4 ndcScaleMat; + ndcScaleMat.scale(ndcXScale, ndcYScale, 1.0F); + + proj = ndcScaleMat * orthoProj; +#else + CC_UNUSED_PARAM(camera); + CC_UNUSED_PARAM(preTransform); + CC_UNUSED_PARAM(proj); + CC_UNUSED_PARAM(view); +#endif +} + + #if CC_USE_XR extern "C" JNIEXPORT void JNICALL Java_com_cocos_lib_xr_CocosXRApi_onAdbCmd(JNIEnv * /*env*/, jobject /*thiz*/, jstring key, jstring value) { diff --git a/native/cocos/platform/java/modules/XRInterface.h b/native/cocos/platform/java/modules/XRInterface.h index 9499e0c6d0..49b81cabbc 100644 --- a/native/cocos/platform/java/modules/XRInterface.h +++ b/native/cocos/platform/java/modules/XRInterface.h @@ -99,6 +99,7 @@ public: xr::XREye getXREyeByRenderWindow(void *window) override; void bindXREyeWithRenderWindow(void *window, xr::XREye eye) override; void handleAppCommand(int appCmd) override; + void adaptOrthographicMatrix(cc::scene::Camera *camera, const ccstd::array &preTransform, Mat4 &proj, Mat4 &view) override; private: void loadAssetsImage(const std::string &imageInfo); void dispatchGamepadEventInternal(const xr::XRControllerEvent &xrControllerEvent); diff --git a/native/cocos/scene/Camera.cpp b/native/cocos/scene/Camera.cpp index d156894109..1984e553f1 100644 --- a/native/cocos/scene/Camera.cpp +++ b/native/cocos/scene/Camera.cpp @@ -210,15 +210,8 @@ void Camera::update(bool forceUpdate /*false*/) { const auto &projFloat = _xr->getXRViewProjectionData(static_cast(wndXREye), _nearClip, _farClip); std::memcpy(_matProj.m, projFloat.data(), sizeof(float) * 16); } else { - const auto &xrFov = _xr->getXREyeFov(static_cast(wndXREye)); - const float left = _orthoHeight * tanf(xrFov[0]); - const float right = _orthoHeight * tanf(xrFov[1]); - const float bottom = _orthoHeight * tanf(xrFov[2]); - const float top = _orthoHeight * tanf(xrFov[3]); - const float projectionSignY = _device->getCapabilities().clipSpaceSignY; - Mat4::createOrthographicOffCenter(left, right, bottom, top, _nearClip, _farClip, - _device->getCapabilities().clipSpaceMinZ, projectionSignY, - static_cast(orientation), &_matProj); + const ccstd::array &preTransform = PRE_TRANSFORMS[static_cast(orientation)]; + _xr->adaptOrthographicMatrix(this, preTransform, _matProj, _matView); } _matProjInv = _matProj.getInversed(); viewProjDirty = true;