diff --git a/native/cocos/platform/interfaces/modules/XRCommon.h b/native/cocos/platform/interfaces/modules/XRCommon.h index 4a0a348270..f05efca767 100644 --- a/native/cocos/platform/interfaces/modules/XRCommon.h +++ b/native/cocos/platform/interfaces/modules/XRCommon.h @@ -115,6 +115,8 @@ enum class XRConfigKey { SPATIAL_MESHING = 51, SPATIAL_MESHING_DATA = 52, SPATIAL_MESHING_SUPPORT_STATUS = 53, + ASYNC_LOAD_ASSETS_IMAGE = 55, + ASYNC_LOAD_ASSETS_IMAGE_RESULTS = 56, MAX_COUNT }; diff --git a/native/cocos/platform/java/modules/XRInterface.cpp b/native/cocos/platform/java/modules/XRInterface.cpp index 6d0e788894..7697190bda 100644 --- a/native/cocos/platform/java/modules/XRInterface.cpp +++ b/native/cocos/platform/java/modules/XRInterface.cpp @@ -460,8 +460,22 @@ void XRInterface::initialize(void *javaVM, void *activity) { std::string imageInfo = value.getString(); _gThreadPool->pushTask([imageInfo, this](int /*tid*/) { - this->loadAssetsImage(imageInfo); + this->loadImageTrackingData(imageInfo); }); + } else if (key == xr::XRConfigKey::ASYNC_LOAD_ASSETS_IMAGE && value.isString()) { + std::string imagePath = value.getString(); + if (imagePath.length() == 0) { + return; + } + + if (!_gThreadPool) { + _gThreadPool = LegacyThreadPool::newSingleThreadPool(); + } + _gThreadPool->pushTask([imagePath, this](int /*tid*/) { + this->asyncLoadAssetsImage(imagePath); + }); + } else if (key == xr::XRConfigKey::ASYNC_LOAD_ASSETS_IMAGE && value.isInt()) { + _isFlipPixelY = value.getInt() == static_cast(gfx::API::GLES3); } }); #if XR_OEM_PICO @@ -964,7 +978,7 @@ void XRInterface::bindXREyeWithRenderWindow(void *window, xr::XREye eye) { } } -void XRInterface::loadAssetsImage(const std::string &imageInfo) { +void XRInterface::loadImageTrackingData(const std::string &imageInfo) { // name|@assets/TrackingImage_SpacesTown.png|0.18|0.26 ccstd::vector segments = StringUtil::split(imageInfo, "|"); std::string imageName = segments.at(0); @@ -975,7 +989,7 @@ void XRInterface::loadAssetsImage(const std::string &imageInfo) { spaceTownImage->addRef(); bool res = spaceTownImage->initWithImageFile(imagePath); if (!res) { - CC_LOG_ERROR("[XRInterface] loadAssetsImage init failed, %s!!!", imageInfo.c_str()); + CC_LOG_ERROR("[XRInterface] loadImageTrackingData init failed, %s!!!", imageInfo.c_str()); return; } uint32_t imageWidth = spaceTownImage->getWidth(); @@ -998,7 +1012,7 @@ void XRInterface::loadAssetsImage(const std::string &imageInfo) { auto app = CC_CURRENT_APPLICATION(); if (!app) { - CC_LOG_ERROR("[XRInterface] loadAssetsImage callback failed, application not exist!!!"); + CC_LOG_ERROR("[XRInterface] loadImageTrackingData callback failed, application not exist!!!"); return; } auto engine = app->getEngine(); @@ -1093,6 +1107,85 @@ void XRInterface::adaptOrthographicMatrix(cc::scene::Camera *camera, const ccstd #endif } +void XRInterface::asyncLoadAssetsImage(const std::string &imagePath) { + auto *assetsImage = new Image(); + assetsImage->addRef(); + bool res = assetsImage->initWithImageFile(imagePath); + if (!res) { + CC_LOG_ERROR("[XRInterface] async load assets image init failed, %s!!!", imagePath.c_str()); + assetsImage->release(); + return; + } + uint32_t imageWidth = assetsImage->getWidth(); + uint32_t imageHeight = assetsImage->getHeight(); + uint32_t bufferSize = assetsImage->getDataLen(); + uint8_t *buffer = nullptr; + if (assetsImage->getRenderFormat() == gfx::Format::RGB8) { + // convert to rgba8 + bufferSize = imageWidth * imageHeight * 4; + buffer = new uint8_t[bufferSize]; + for (uint32_t y = 0; y < imageHeight; y++) { + for (uint32_t x = 0; x < imageWidth; x++) { + const unsigned int pixel = x + y * imageWidth; + const unsigned int pixelFlip = x + (imageHeight - y - 1) * imageWidth; + const uint8_t *originalPixel = &assetsImage->getData()[static_cast(pixel * 3)]; + uint8_t *convertedPixel = nullptr; + if (_isFlipPixelY) { + convertedPixel = &buffer[static_cast(pixelFlip * 4)]; + } else { + convertedPixel = &buffer[static_cast(pixel * 4)]; + } + convertedPixel[0] = originalPixel[0]; + convertedPixel[1] = originalPixel[1]; + convertedPixel[2] = originalPixel[2]; + convertedPixel[3] = 255.0F; + } + } + } else { + buffer = new uint8_t[bufferSize]; + if(_isFlipPixelY) { + for (uint32_t y = 0; y < imageHeight; y++) { + for (uint32_t x = 0; x < imageWidth; x++) { + const unsigned int pixel = x + y * imageWidth; + const unsigned int pixelFlip = x + (imageHeight - y - 1) * imageWidth; + const uint8_t *originalPixel = &assetsImage->getData()[static_cast(pixel * 4)]; + uint8_t *convertedPixel = &buffer[static_cast(pixelFlip * 4)]; + convertedPixel[0] = originalPixel[0]; + convertedPixel[1] = originalPixel[1]; + convertedPixel[2] = originalPixel[2]; + convertedPixel[3] = originalPixel[3]; + } + } + } else { + memcpy(buffer, assetsImage->getData(), bufferSize); + } + } + auto app = CC_CURRENT_APPLICATION(); + if (!app) { + CC_LOG_ERROR("[XRInterface] loadAssetsImage callback failed, application not exist!!!"); + return; + } + auto engine = app->getEngine(); + CC_ASSERT_NOT_NULL(engine); + engine->getScheduler()->performFunctionInCocosThread([=]() { + auto *imageData = new xr::XRTrackingImageData(); + imageData->friendlyName = imagePath; + imageData->bufferSize = bufferSize; + imageData->buffer = buffer; + imageData->pixelSizeWidth = imageWidth; + imageData->pixelSizeHeight = imageHeight; + if (!this->getXRConfig(xr::XRConfigKey::ASYNC_LOAD_ASSETS_IMAGE_RESULTS).getPointer()) { + auto *imagesMapPtr = new std::unordered_map(); + (*imagesMapPtr).emplace(std::make_pair(imagePath, static_cast(imageData))); + this->setXRConfig(xr::XRConfigKey::ASYNC_LOAD_ASSETS_IMAGE_RESULTS, static_cast(imagesMapPtr)); + } else { + auto *imagesMapPtr = static_cast *>(getXRConfig(xr::XRConfigKey::ASYNC_LOAD_ASSETS_IMAGE_RESULTS).getPointer()); + (*imagesMapPtr).emplace(std::make_pair(imagePath, static_cast(imageData))); + } + }); + assetsImage->release(); +} #if CC_USE_XR diff --git a/native/cocos/platform/java/modules/XRInterface.h b/native/cocos/platform/java/modules/XRInterface.h index 49b81cabbc..af4e5cebd1 100644 --- a/native/cocos/platform/java/modules/XRInterface.h +++ b/native/cocos/platform/java/modules/XRInterface.h @@ -101,7 +101,8 @@ public: void handleAppCommand(int appCmd) override; void adaptOrthographicMatrix(cc::scene::Camera *camera, const ccstd::array &preTransform, Mat4 &proj, Mat4 &view) override; private: - void loadAssetsImage(const std::string &imageInfo); + void loadImageTrackingData(const std::string &imageInfo); + void asyncLoadAssetsImage(const std::string &imagePath); void dispatchGamepadEventInternal(const xr::XRControllerEvent &xrControllerEvent); void dispatchHandleEventInternal(const xr::XRControllerEvent &xrControllerEvent); void dispatchHMDEventInternal(const xr::XRControllerEvent &xrControllerEvent); @@ -131,6 +132,7 @@ private: cc::IntrusivePtr _xrRemotePreviewManager{nullptr}; #endif LegacyThreadPool *_gThreadPool{nullptr}; + bool _isFlipPixelY{false}; }; } // namespace cc