From e459f74ae95f68b86e172273b924ac69d8660a9f Mon Sep 17 00:00:00 2001 From: pandamicro Date: Wed, 11 Jan 2017 19:03:13 +0800 Subject: [PATCH 1/6] Fix AssetsManager issues collected from the community --- .../assets-manager/AssetsManagerEx.cpp | 129 +++++++++--------- .../assets-manager/AssetsManagerEx.h | 7 +- native/extensions/assets-manager/Manifest.cpp | 2 +- 3 files changed, 64 insertions(+), 74 deletions(-) diff --git a/native/extensions/assets-manager/AssetsManagerEx.cpp b/native/extensions/assets-manager/AssetsManagerEx.cpp index 991911c2e3..f1e22b50bc 100644 --- a/native/extensions/assets-manager/AssetsManagerEx.cpp +++ b/native/extensions/assets-manager/AssetsManagerEx.cpp @@ -420,18 +420,42 @@ bool AssetsManagerEx::decompress(const std::string &zip) return true; } -void AssetsManagerEx::decompressDownloadedZip() +void AssetsManagerEx::decompressDownloadedZip(const std::string &customId, const std::string &storagePath) { - // Decompress all compressed files - for (auto it = _compressedFiles.begin(); it != _compressedFiles.end(); ++it) { - std::string zipfile = *it; - if (!decompress(zipfile)) + struct AsyncData + { + std::string customId; + std::string zipFile; + bool succeed; + }; + + AsyncData* asyncData = new AsyncData; + asyncData->customId = customId; + asyncData->zipFile = storagePath; + asyncData->succeed = false; + + std::function decompressFinished = [this](void* param) { + auto dataInner = reinterpret_cast(param); + if (dataInner->succeed) { - dispatchUpdateEvent(EventAssetsManagerEx::EventCode::ERROR_DECOMPRESS, "", "Unable to decompress file " + zipfile); + fileSuccess(dataInner->customId, dataInner->zipFile); } - _fileUtils->removeFile(zipfile); - } - _compressedFiles.clear(); + else + { + std::string errorMsg = "Unable to decompress file " + dataInner->zipFile; + dispatchUpdateEvent(EventAssetsManagerEx::EventCode::ERROR_DECOMPRESS, "", errorMsg); + fileError(dataInner->customId, errorMsg); + } + delete dataInner; + }; + AsyncTaskPool::getInstance()->enqueue(AsyncTaskPool::TaskType::TASK_OTHER, decompressFinished, (void*)asyncData, [this, asyncData]() { + // Decompress all compressed files + if (decompress(asyncData->zipFile)) + { + asyncData->succeed = true; + } + _fileUtils->removeFile(asyncData->zipFile); + }); } void AssetsManagerEx::dispatchUpdateEvent(EventAssetsManagerEx::EventCode code, const std::string &assetId/* = ""*/, const std::string &message/* = ""*/, int curle_code/* = CURLE_OK*/, int curlm_code/* = CURLM_OK*/) @@ -515,14 +539,11 @@ void AssetsManagerEx::parseVersion() } else { - // Save the old update entry type since its value may be changed in event listener. - UpdateEntry oldUpdateEntry = _updateEntry; - _updateState = State::NEED_UPDATE; dispatchUpdateEvent(EventAssetsManagerEx::EventCode::NEW_VERSION_FOUND); // Wait to update so continue the process - if (oldUpdateEntry == UpdateEntry::DO_UPDATE) + if (_updateEntry == UpdateEntry::DO_UPDATE) { _updateState = State::PREDOWNLOAD_MANIFEST; downloadManifest(); @@ -600,7 +621,6 @@ void AssetsManagerEx::startUpdate() // Clean up before update _failedUnits.clear(); _downloadUnits.clear(); - _compressedFiles.clear(); _totalWaitToDownload = _totalToDownload = 0; _percent = _percentByFile = _sizeCollected = _totalSize = 0; _downloadedSize.clear(); @@ -668,6 +688,9 @@ void AssetsManagerEx::startUpdate() _tempManifest->setAssetDownloadState(key, Manifest::DownloadState::SUCCESSED); } } + // Save current download manifest information for resuming + _tempManifest->saveToFile(_tempManifestPath); + _totalWaitToDownload = _totalToDownload = (int)_downloadUnits.size(); this->batchDownload(); @@ -689,50 +712,10 @@ void AssetsManagerEx::updateSucceed() _remoteManifest = nullptr; // 3. make local manifest take effect prepareLocalManifest(); - - - _updateState = State::UNZIPPING; - // 4. decompress all compressed files - //decompressDownloadedZip(); - - struct AsyncData - { - std::vector compressedFiles; - std::string errorCompressedFile; - }; - - AsyncData* asyncData = new AsyncData; - asyncData->compressedFiles = _compressedFiles; - _compressedFiles.clear(); - - std::function mainThread = [this](void* param) { - auto asyncDataInner = reinterpret_cast(param); - if (asyncDataInner->errorCompressedFile.empty()) - { - // 5. Set update state - _updateState = State::UP_TO_DATE; - // 6. Notify finished event - dispatchUpdateEvent(EventAssetsManagerEx::EventCode::UPDATE_FINISHED); - } - else - { - _updateState = State::FAIL_TO_UPDATE; - dispatchUpdateEvent(EventAssetsManagerEx::EventCode::ERROR_DECOMPRESS, "", "Unable to decompress file " + asyncDataInner->errorCompressedFile); - } - - delete asyncDataInner; - }; - AsyncTaskPool::getInstance()->enqueue(AsyncTaskPool::TaskType::TASK_OTHER, mainThread, (void*)asyncData, [this, asyncData]() { - // Decompress all compressed files - for (auto& zipFile : asyncData->compressedFiles) { - if (!decompress(zipFile)) - { - asyncData->errorCompressedFile = zipFile; - break; - } - _fileUtils->removeFile(zipFile); - } - }); + // 4. Set update state + _updateState = State::UP_TO_DATE; + // 5. Notify finished event + dispatchUpdateEvent(EventAssetsManagerEx::EventCode::UPDATE_FINISHED); } void AssetsManagerEx::checkUpdate() @@ -909,16 +892,11 @@ void AssetsManagerEx::fileError(const std::string& identifier, const std::string queueDowload(); } -void AssetsManagerEx::fileSuccess(const std::string &customId, const std::string &storagePath, bool compressed) +void AssetsManagerEx::fileSuccess(const std::string &customId, const std::string &storagePath) { // Set download state to SUCCESSED _tempManifest->setAssetDownloadState(customId, Manifest::DownloadState::SUCCESSED); - // Add file to need decompress list - if (compressed) { - _compressedFiles.push_back(storagePath); - } - auto unitIt = _failedUnits.find(customId); // Found unit and delete it if (unitIt != _failedUnits.end()) @@ -1051,7 +1029,14 @@ void AssetsManagerEx::onSuccess(const std::string &/*srcUrl*/, const std::string if (ok) { bool compressed = assetIt != assets.end() ? assetIt->second.compressed : false; - fileSuccess(customId, storagePath, compressed); + if (compressed) + { + decompressDownloadedZip(customId, storagePath); + } + else + { + fileSuccess(customId, storagePath); + } } else { @@ -1091,12 +1076,14 @@ void AssetsManagerEx::batchDownload() void AssetsManagerEx::queueDowload() { - if (_queue.size() == 0) + if (_totalWaitToDownload == 0) { this->onDownloadUnitsFinished(); return; } + bool taskCreated = false; + while (_currConcurrentTask < _maxConcurrentTask && _queue.size() > 0) { std::string key = _queue.back(); @@ -1105,6 +1092,14 @@ void AssetsManagerEx::queueDowload() _currConcurrentTask++; DownloadUnit& unit = _downloadUnits[key]; _downloader->createDownloadFileTask(unit.srcUrl, unit.storagePath, unit.customId); + + _tempManifest->setAssetDownloadState(key, Manifest::DownloadState::DOWNLOADING); + taskCreated = true; + } + if (taskCreated) + { + // Save current download manifest information for resuming + _tempManifest->saveToFile(_tempManifestPath); } } @@ -1116,8 +1111,6 @@ void AssetsManagerEx::onDownloadUnitsFinished() // Save current download manifest information for resuming _tempManifest->saveToFile(_tempManifestPath); - decompressDownloadedZip(); - _updateState = State::FAIL_TO_UPDATE; dispatchUpdateEvent(EventAssetsManagerEx::EventCode::UPDATE_FAILED); } diff --git a/native/extensions/assets-manager/AssetsManagerEx.h b/native/extensions/assets-manager/AssetsManagerEx.h index a160ff97c4..78c6efad24 100644 --- a/native/extensions/assets-manager/AssetsManagerEx.h +++ b/native/extensions/assets-manager/AssetsManagerEx.h @@ -157,7 +157,7 @@ protected: void startUpdate(); void updateSucceed(); bool decompress(const std::string &filename); - void decompressDownloadedZip(); + void decompressDownloadedZip(const std::string &customId, const std::string &storagePath); /** @brief Update a list of assets under the current AssetsManagerEx context */ @@ -177,7 +177,7 @@ protected: void fileError(const std::string& identifier, const std::string& errorStr, int errorCode = 0, int errorCodeInternal = 0); - void fileSuccess(const std::string &customId, const std::string &storagePath, bool compressed); + void fileSuccess(const std::string &customId, const std::string &storagePath); /** @brief Call back function for error handling, the error will then be reported to user's listener registed in addUpdateEventListener @@ -285,9 +285,6 @@ private: //! Current concurrent task count int _currConcurrentTask; - //! All files to be decompressed - std::vector _compressedFiles; - //! Download percent float _percent; diff --git a/native/extensions/assets-manager/Manifest.cpp b/native/extensions/assets-manager/Manifest.cpp index 96f14baef7..b9e5d2af1f 100644 --- a/native/extensions/assets-manager/Manifest.cpp +++ b/native/extensions/assets-manager/Manifest.cpp @@ -422,7 +422,7 @@ Manifest::Asset Manifest::parseAsset(const std::string &path, const rapidjson::V if ( json.HasMember(KEY_SIZE) && json[KEY_SIZE].IsFloat() ) { - asset.size = json[KEY_COMPRESSED].GetFloat(); + asset.size = json[KEY_SIZE].GetFloat(); } else asset.size = 0; From 16b406ac5894f8989ef5878c85b23a97379010c2 Mon Sep 17 00:00:00 2001 From: pandamicro Date: Thu, 12 Jan 2017 09:50:59 +0800 Subject: [PATCH 2/6] Manifest basic version compare support string versions --- .../assets-manager/AssetsManagerEx.h | 4 +-- native/extensions/assets-manager/Manifest.cpp | 28 +++++++++++++++++-- native/extensions/assets-manager/Manifest.h | 2 +- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/native/extensions/assets-manager/AssetsManagerEx.h b/native/extensions/assets-manager/AssetsManagerEx.h index 78c6efad24..a9b5e6e3d6 100644 --- a/native/extensions/assets-manager/AssetsManagerEx.h +++ b/native/extensions/assets-manager/AssetsManagerEx.h @@ -119,7 +119,7 @@ public: /** @brief Set the handle function for comparing manifests versions * @param handle The compare function */ - void setVersionCompareHandle(const std::function& handle) {_versionCompareHandle = handle;}; + void setVersionCompareHandle(const std::function& handle) {_versionCompareHandle = handle;}; /** @brief Set the verification function for checking whether downloaded asset is correct, e.g. using md5 verification * @param callback The verify callback function @@ -309,7 +309,7 @@ private: int _totalWaitToDownload; //! Handle function to compare versions between different manifests - std::function _versionCompareHandle; + std::function _versionCompareHandle; //! Callback function to verify the downloaded assets std::function _verifyCallback; diff --git a/native/extensions/assets-manager/Manifest.cpp b/native/extensions/assets-manager/Manifest.cpp index b9e5d2af1f..9696da83f8 100644 --- a/native/extensions/assets-manager/Manifest.cpp +++ b/native/extensions/assets-manager/Manifest.cpp @@ -27,6 +27,7 @@ #include "json/stringbuffer.h" #include +#include #define KEY_VERSION "version" #define KEY_PACKAGE_URL "packageUrl" @@ -48,6 +49,27 @@ NS_CC_EXT_BEGIN +static int cmpVersion(const std::string& v1, const std::string& v2) +{ + int i; + int oct_v1[4] = {0}, oct_v2[4] = {0}; + int filled1 = std::sscanf(v1.c_str(), "%d.%d.%d.%d", &oct_v1[0], &oct_v1[1], &oct_v1[2], &oct_v1[3]); + int filled2 = std::sscanf(v2.c_str(), "%d.%d.%d.%d", &oct_v2[0], &oct_v2[1], &oct_v2[2], &oct_v2[3]); + + if (filled1 == 0 || filled2 == 0) + { + return strcmp(v1.c_str(), v2.c_str()); + } + for (i = 0; i < 4; i++) + { + if (oct_v1[i] > oct_v2[i]) + return 1; + else if (oct_v1[i] < oct_v2[i]) + return -1; + } + return 0; +} + Manifest::Manifest(const std::string& manifestUrl/* = ""*/) : _versionLoaded(false) , _loaded(false) @@ -157,18 +179,18 @@ bool Manifest::versionEquals(const Manifest *b) const return true; } -bool Manifest::versionGreater(const Manifest *b, const std::function& handle) const +bool Manifest::versionGreater(const Manifest *b, const std::function& handle) const { std::string localVersion = getVersion(); std::string bVersion = b->getVersion(); bool greater; if (handle) { - greater = handle(localVersion, bVersion); + greater = handle(localVersion, bVersion) >= 0; } else { - greater = strcmp(localVersion.c_str(), bVersion.c_str()) >= 0; + greater = cmpVersion(localVersion, bVersion) >= 0; } return greater; } diff --git a/native/extensions/assets-manager/Manifest.h b/native/extensions/assets-manager/Manifest.h index b98b072a65..b77f5168f9 100644 --- a/native/extensions/assets-manager/Manifest.h +++ b/native/extensions/assets-manager/Manifest.h @@ -145,7 +145,7 @@ protected: * @param [handle] Customized comparasion handle function * @return Greater or not */ - bool versionGreater(const Manifest *b, const std::function& handle) const; + bool versionGreater(const Manifest *b, const std::function& handle) const; /** @brief Generate difference between this Manifest and another. * @param b The other manifest From 1294fa9ee85dba9c80ec09ac2d33564823c1041e Mon Sep 17 00:00:00 2001 From: pandamicro Date: Thu, 12 Jan 2017 12:48:51 +0800 Subject: [PATCH 3/6] Ensure loading manifest in app package correctly --- native/extensions/assets-manager/AssetsManagerEx.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/native/extensions/assets-manager/AssetsManagerEx.cpp b/native/extensions/assets-manager/AssetsManagerEx.cpp index f1e22b50bc..6510f4791a 100644 --- a/native/extensions/assets-manager/AssetsManagerEx.cpp +++ b/native/extensions/assets-manager/AssetsManagerEx.cpp @@ -199,7 +199,13 @@ void AssetsManagerEx::loadLocalManifest(const std::string& /*manifestUrl*/) } // Load local manifest in app package + std::vector searchPaths = _fileUtils->getSearchPaths(); + // Ensure no search path is used to load this manifest + std::vector emptyPaths = {""}; + _fileUtils->setSearchPaths(emptyPaths); _localManifest->parse(_manifestUrl); + // Restore search paths + _fileUtils->setSearchPaths(searchPaths); if (_localManifest->isLoaded()) { // Compare with cached manifest to determine which one to use From 7a9e3a3b1d4895950a86755af1d4522728d12b7d Mon Sep 17 00:00:00 2001 From: pandamicro Date: Fri, 13 Jan 2017 10:28:24 +0800 Subject: [PATCH 4/6] Add FileUtils::listFiles and FileUtils::listFilesRecursively --- .../cocos2d_libs.xcodeproj/project.pbxproj | 19 +++++ native/cocos/platform/CCFileUtils.cpp | 82 +++++++++++++++++++ native/cocos/platform/CCFileUtils.h | 16 ++++ native/external/config.json | 2 +- 4 files changed, 118 insertions(+), 1 deletion(-) diff --git a/native/build/cocos2d_libs.xcodeproj/project.pbxproj b/native/build/cocos2d_libs.xcodeproj/project.pbxproj index 7e74730551..6edeaaa26e 100644 --- a/native/build/cocos2d_libs.xcodeproj/project.pbxproj +++ b/native/build/cocos2d_libs.xcodeproj/project.pbxproj @@ -1255,6 +1255,8 @@ BA68D7BF1D62F71700B7A3F9 /* CCFrameBuffer.h in Headers */ = {isa = PBXBuildFile; fileRef = BA68D7BB1D62F71700B7A3F9 /* CCFrameBuffer.h */; }; BA8122AB1D77C4150010BEE9 /* CCLabelTTF.cpp in Sources */ = {isa = PBXBuildFile; fileRef = BA8122AA1D77C4150010BEE9 /* CCLabelTTF.cpp */; }; BA8122AC1D77C4160010BEE9 /* CCLabelTTF.cpp in Sources */ = {isa = PBXBuildFile; fileRef = BA8122AA1D77C4150010BEE9 /* CCLabelTTF.cpp */; }; + BAEA45551E279D5C00FA219F /* tinydir.h in Headers */ = {isa = PBXBuildFile; fileRef = BAEA45541E279D5C00FA219F /* tinydir.h */; }; + BAEA45561E279D5C00FA219F /* tinydir.h in Headers */ = {isa = PBXBuildFile; fileRef = BAEA45541E279D5C00FA219F /* tinydir.h */; }; BAFF7C721D59E0DA0051B92F /* CCActionFrame.cpp in Sources */ = {isa = PBXBuildFile; fileRef = BAFF7C321D59E0DA0051B92F /* CCActionFrame.cpp */; }; BAFF7C731D59E0DA0051B92F /* CCActionFrame.cpp in Sources */ = {isa = PBXBuildFile; fileRef = BAFF7C321D59E0DA0051B92F /* CCActionFrame.cpp */; }; BAFF7C741D59E0DA0051B92F /* CCActionFrame.h in Headers */ = {isa = PBXBuildFile; fileRef = BAFF7C331D59E0DA0051B92F /* CCActionFrame.h */; }; @@ -2410,6 +2412,9 @@ BA68D7BA1D62F71700B7A3F9 /* CCFrameBuffer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CCFrameBuffer.cpp; sourceTree = ""; }; BA68D7BB1D62F71700B7A3F9 /* CCFrameBuffer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CCFrameBuffer.h; sourceTree = ""; }; BA8122AA1D77C4150010BEE9 /* CCLabelTTF.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CCLabelTTF.cpp; sourceTree = ""; }; + BAEA45521E279D5C00FA219F /* COPYING */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = COPYING; sourceTree = ""; }; + BAEA45531E279D5C00FA219F /* package.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = package.json; sourceTree = ""; }; + BAEA45541E279D5C00FA219F /* tinydir.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = tinydir.h; sourceTree = ""; }; BAFF7C321D59E0DA0051B92F /* CCActionFrame.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CCActionFrame.cpp; sourceTree = ""; }; BAFF7C331D59E0DA0051B92F /* CCActionFrame.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CCActionFrame.h; sourceTree = ""; }; BAFF7C341D59E0DA0051B92F /* CCActionFrameEasing.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CCActionFrameEasing.cpp; sourceTree = ""; }; @@ -3094,6 +3099,7 @@ 1A57033E180BD0490088DEC7 /* external */ = { isa = PBXGroup; children = ( + BAEA45511E279D5C00FA219F /* tinydir */, BA68D7951D62F4B600B7A3F9 /* clipper */, BA68D76E1D62F4A500B7A3F9 /* poly2tri */, ED3057951BEC779A0083C3ED /* ConvertUTF */, @@ -4090,6 +4096,17 @@ path = ../external/sources/clipper; sourceTree = ""; }; + BAEA45511E279D5C00FA219F /* tinydir */ = { + isa = PBXGroup; + children = ( + BAEA45521E279D5C00FA219F /* COPYING */, + BAEA45531E279D5C00FA219F /* package.json */, + BAEA45541E279D5C00FA219F /* tinydir.h */, + ); + name = tinydir; + path = ../external/sources/tinydir; + sourceTree = ""; + }; C50306621B60B56C001E6D43 /* Skeleton */ = { isa = PBXGroup; children = ( @@ -4420,6 +4437,7 @@ 50CB247D19D9C5A100687767 /* AudioPlayer.h in Headers */, 1A570114180BC8EE0088DEC7 /* CCDrawNode.h in Headers */, 15AE188319AAD33D00C27E9E /* CCBSelectorResolver.h in Headers */, + BAEA45551E279D5C00FA219F /* tinydir.h in Headers */, 15AE1B5819AADA9900C27E9E /* UISlider.h in Headers */, 1A57011D180BC90D0088DEC7 /* CCGrabber.h in Headers */, 15AE1B6819AADA9900C27E9E /* UIScale9Sprite.h in Headers */, @@ -5029,6 +5047,7 @@ 5034CA4A191D591100CE6051 /* ccShader_Label_df.frag in Headers */, 4D2E690A1CC543B40032DD5C /* CCGraphicsNode.h in Headers */, BA68D7901D62F4A500B7A3F9 /* sweep.h in Headers */, + BAEA45561E279D5C00FA219F /* tinydir.h in Headers */, BAFF7CCB1D59E0DB0051B92F /* CCDisplayManager.h in Headers */, 2980F0221BA9A5550059E678 /* CCUIEditBoxIOS.h in Headers */, FA6F1B941D80F858007DD223 /* AnimationData.h in Headers */, diff --git a/native/cocos/platform/CCFileUtils.cpp b/native/cocos/platform/CCFileUtils.cpp index 9341252503..1792def350 100644 --- a/native/cocos/platform/CCFileUtils.cpp +++ b/native/cocos/platform/CCFileUtils.cpp @@ -33,6 +33,8 @@ THE SOFTWARE. #include "platform/CCSAXParser.h" #include "tinyxml2/tinyxml2.h" +#include "tinydir/tinydir.h" + #ifdef MINIZIP_FROM_SYSTEM #include #else // from our embedded sources @@ -1028,6 +1030,86 @@ bool FileUtils::isDirectoryExist(const std::string& dirPath) const return false; } +std::vector FileUtils::listFiles(const std::string& dirPath) const +{ + std::string fullpath = fullPathForFilename(dirPath); + std::vector files; + if (isDirectoryExist(fullpath)) + { + tinydir_dir dir; + if (tinydir_open(&dir, fullpath.c_str()) != -1) + { + while (dir.has_next) + { + tinydir_file file; + if (tinydir_readfile(&dir, &file) == -1) + { + // Error getting file + break; + } + + std::string filepath = file.name; + if (file.is_dir) + { + filepath.append("/"); + } + files.push_back(file.name); + + if (tinydir_next(&dir) == -1) + { + // Error getting next file + break; + } + } + } + tinydir_close(&dir); + } + return files; +} + +void FileUtils::listFilesRecursively(const std::string& dirPath, std::vector *files) const +{ + std::string fullpath = fullPathForFilename(dirPath); + if (isDirectoryExist(fullpath)) + { + tinydir_dir dir; + if (tinydir_open(&dir, fullpath.c_str()) != -1) + { + while (dir.has_next) + { + tinydir_file file; + if (tinydir_readfile(&dir, &file) == -1) + { + // Error getting file + break; + } + + std::string filepath = file.path; + if (file.name[0] != '.') + { + if (file.is_dir) + { + filepath.append("/"); + files->push_back(filepath); + listFilesRecursively(filepath, files); + } + else + { + files->push_back(filepath); + } + } + + if (tinydir_next(&dir) == -1) + { + // Error getting next file + break; + } + } + } + tinydir_close(&dir); + } +} + #if (CC_TARGET_PLATFORM == CC_PLATFORM_WIN32) || (CC_TARGET_PLATFORM == CC_PLATFORM_WINRT) // windows os implement should override in platform specific FileUtiles class bool FileUtils::isDirectoryExistInternal(const std::string& dirPath) const diff --git a/native/cocos/platform/CCFileUtils.h b/native/cocos/platform/CCFileUtils.h index f5cddd0972..617521dd07 100644 --- a/native/cocos/platform/CCFileUtils.h +++ b/native/cocos/platform/CCFileUtils.h @@ -565,6 +565,22 @@ public: * @return True if the directory exists, false if not. */ virtual bool isDirectoryExist(const std::string& dirPath) const; + + /** + * List all files in a directory. + * + * @param dirPath The path of the directory, it could be a relative or an absolute path. + * @return File paths in a string vector + */ + virtual std::vector listFiles(const std::string& dirPath) const; + + /** + * List all files recursively in a directory. + * + * @param dirPath The path of the directory, it could be a relative or an absolute path. + * @return File paths in a string vector + */ + virtual void listFilesRecursively(const std::string& dirPath, std::vector *files) const; /** * Creates a directory. diff --git a/native/external/config.json b/native/external/config.json index fa12146643..cdf4f54364 100644 --- a/native/external/config.json +++ b/native/external/config.json @@ -1,5 +1,5 @@ { - "version":"v3-deps-21", + "version":"v3-deps-22", "zip_file_size":"62588953", "repo_name":"cocos2d-x-lite-external", "repo_parent":"https://github.com/cocos-creator/" From 28ea176e4d060da7f991bf3f03a20355f38f2f4a Mon Sep 17 00:00:00 2001 From: pandamicro Date: Fri, 13 Jan 2017 10:31:32 +0800 Subject: [PATCH 5/6] Refactor AssetsManager update logic - Separate temporary version and local cached version - Create folder when needed - Fix search path pollution when local cached version exists --- .../assets-manager/AssetsManagerEx.cpp | 151 ++++++++++++------ .../assets-manager/AssetsManagerEx.h | 9 +- native/extensions/assets-manager/Manifest.cpp | 14 -- native/extensions/assets-manager/Manifest.h | 6 +- 4 files changed, 110 insertions(+), 70 deletions(-) diff --git a/native/extensions/assets-manager/AssetsManagerEx.cpp b/native/extensions/assets-manager/AssetsManagerEx.cpp index 6510f4791a..1531ec22a8 100644 --- a/native/extensions/assets-manager/AssetsManagerEx.cpp +++ b/native/extensions/assets-manager/AssetsManagerEx.cpp @@ -39,12 +39,13 @@ NS_CC_EXT_BEGIN #define VERSION_FILENAME "version.manifest" #define TEMP_MANIFEST_FILENAME "project.manifest.temp" +#define TEMP_PACKAGE_SUFFIX "_temp" #define MANIFEST_FILENAME "project.manifest" #define BUFFER_SIZE 8192 #define MAX_FILENAME 512 -#define DEFAULT_CONNECTION_TIMEOUT 8 +#define DEFAULT_CONNECTION_TIMEOUT 16 const std::string AssetsManagerEx::VERSION_ID = "@version"; const std::string AssetsManagerEx::MANIFEST_ID = "@manifest"; @@ -55,7 +56,7 @@ AssetsManagerEx::AssetsManagerEx(const std::string& manifestUrl, const std::stri : _updateState(State::UNCHECKED) , _assets(nullptr) , _storagePath("") -, _cacheVersionPath("") +, _tempVersionPath("") , _cacheManifestPath("") , _tempManifestPath("") , _manifestUrl(manifestUrl) @@ -79,7 +80,13 @@ AssetsManagerEx::AssetsManagerEx(const std::string& manifestUrl, const std::stri _eventName = EventListenerAssetsManagerEx::LISTENER_ID + pointer; _fileUtils = FileUtils::getInstance(); - _downloader = std::shared_ptr(new network::Downloader); + network::DownloaderHints hints = + { + static_cast(_maxConcurrentTask), + DEFAULT_CONNECTION_TIMEOUT, + ".tmp" + }; + _downloader = std::shared_ptr(new network::Downloader(hints)); // _downloader->setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT); _downloader->onTaskError = std::bind(&AssetsManagerEx::onError, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4); _downloader->onTaskProgress = [this](const network::DownloadTask& task, @@ -94,9 +101,9 @@ AssetsManagerEx::AssetsManagerEx(const std::string& manifestUrl, const std::stri this->onSuccess(task.requestURL, task.storagePath, task.identifier); }; setStoragePath(storagePath); - _cacheVersionPath = _storagePath + VERSION_FILENAME; + _tempVersionPath = _tempStoragePath + VERSION_FILENAME; _cacheManifestPath = _storagePath + MANIFEST_FILENAME; - _tempManifestPath = _storagePath + TEMP_MANIFEST_FILENAME; + _tempManifestPath = _tempStoragePath + TEMP_MANIFEST_FILENAME; initManifests(manifestUrl); } @@ -141,9 +148,16 @@ void AssetsManagerEx::initManifests(const std::string& manifestUrl) if (_tempManifest) { _tempManifest->parse(_tempManifestPath); - if (!_tempManifest->isLoaded() && _fileUtils->isFileExist(_tempManifestPath)) + // Previous update is interrupted + if (_fileUtils->isFileExist(_tempManifestPath)) { - _fileUtils->removeFile(_tempManifestPath); + // Manifest parse failed, remove all temp files + if (!_tempManifest->isLoaded()) + { + _fileUtils->removeDirectory(_tempStoragePath); + CC_SAFE_RELEASE(_tempManifest); + _tempManifest = nullptr; + } } } else @@ -165,9 +179,12 @@ void AssetsManagerEx::initManifests(const std::string& manifestUrl) if (!_inited) { - CC_SAFE_DELETE(_localManifest); - CC_SAFE_DELETE(_tempManifest); - CC_SAFE_DELETE(_remoteManifest); + CC_SAFE_RELEASE(_localManifest); + CC_SAFE_RELEASE(_tempManifest); + CC_SAFE_RELEASE(_remoteManifest); + _localManifest = nullptr; + _tempManifest = nullptr; + _remoteManifest = nullptr; } } @@ -278,6 +295,10 @@ void AssetsManagerEx::setStoragePath(const std::string& storagePath) _storagePath = storagePath; adjustPath(_storagePath); _fileUtils->createDirectory(_storagePath); + + _tempStoragePath = _storagePath; + _tempStoragePath.insert(_storagePath.size() - 1, TEMP_PACKAGE_SUFFIX); + _fileUtils->createDirectory(_tempStoragePath); } void AssetsManagerEx::adjustPath(std::string &path) @@ -512,7 +533,7 @@ void AssetsManagerEx::downloadVersion() { _updateState = State::DOWNLOADING_VERSION; // Download version file asynchronously - _downloader->createDownloadFileTask(versionUrl, _cacheVersionPath, VERSION_ID); + _downloader->createDownloadFileTask(versionUrl, _tempVersionPath, VERSION_ID); } // No version file found else @@ -528,7 +549,7 @@ void AssetsManagerEx::parseVersion() if (_updateState != State::VERSION_LOADED) return; - _remoteManifest->parseVersion(_cacheVersionPath); + _remoteManifest->parseVersion(_tempVersionPath); if (!_remoteManifest->isVersionLoaded()) { @@ -541,6 +562,7 @@ void AssetsManagerEx::parseVersion() if (_localManifest->versionGreater(_remoteManifest, _versionCompareHandle)) { _updateState = State::UP_TO_DATE; + _fileUtils->removeDirectory(_tempStoragePath); dispatchUpdateEvent(EventAssetsManagerEx::EventCode::ALREADY_UP_TO_DATE); } else @@ -603,6 +625,7 @@ void AssetsManagerEx::parseManifest() if (_localManifest->versionGreater(_remoteManifest, _versionCompareHandle)) { _updateState = State::UP_TO_DATE; + _fileUtils->removeDirectory(_tempStoragePath); dispatchUpdateEvent(EventAssetsManagerEx::EventCode::ALREADY_UP_TO_DATE); } else @@ -633,8 +656,9 @@ void AssetsManagerEx::startUpdate() _totalEnabled = false; // Temporary manifest exists, resuming previous download - if (_tempManifest->isLoaded() && _tempManifest->versionEquals(_remoteManifest)) + if (_tempManifest && _tempManifest->isLoaded() && _tempManifest->versionEquals(_remoteManifest)) { + _tempManifest->saveToFile(_tempManifestPath); _tempManifest->genResumeAssetsList(&_downloadUnits); _totalWaitToDownload = _totalToDownload = (int)_downloadUnits.size(); this->batchDownload(); @@ -642,16 +666,24 @@ void AssetsManagerEx::startUpdate() std::string msg = StringUtils::format("Resuming from previous unfinished update, %d files remains to be finished.", _totalToDownload); dispatchUpdateEvent(EventAssetsManagerEx::EventCode::UPDATE_PROGRESSION, "", msg); } - // Check difference else { - // Remove all temp files - _tempManifest->cleanupDownloadingAssets(); - _tempManifest->release(); + // Temporary manifest exists, but can't be parsed or version doesn't equals remote manifest (out of date) + if (_tempManifest) + { + // Remove all temp files + _fileUtils->removeDirectory(_tempStoragePath); + CC_SAFE_RELEASE(_tempManifest); + // Recreate temp storage path and save remote manifest + _fileUtils->createDirectory(_tempStoragePath); + _remoteManifest->saveToFile(_tempManifestPath); + } + // Temporary manifest will be used to register the download states of each asset, // in this case, it equals remote manifest. _tempManifest = _remoteManifest; + // Check difference between local manifest and remote manifest std::unordered_map diff_map = _localManifest->genDiff(_remoteManifest); if (diff_map.size() == 0) { @@ -661,28 +693,6 @@ void AssetsManagerEx::startUpdate() { // Generate download units for all assets that need to be updated or added std::string packageUrl = _remoteManifest->getPackageUrl(); - for (auto it = diff_map.begin(); it != diff_map.end(); ++it) - { - Manifest::AssetDiff diff = it->second; - - if (diff.type == Manifest::DiffType::DELETED) - { - _fileUtils->removeFile(_storagePath + diff.asset.path); - } - else - { - std::string path = diff.asset.path; - // Create path - _fileUtils->createDirectory(basename(_storagePath + path)); - - DownloadUnit unit; - unit.customId = it->first; - unit.srcUrl = packageUrl + path; - unit.storagePath = _storagePath + path; - unit.size = diff.asset.size; - _downloadUnits.emplace(unit.customId, unit); - } - } // Set other assets' downloadState to SUCCEED auto &assets = _remoteManifest->getAssets(); for (auto it = assets.cbegin(); it != assets.cend(); ++it) @@ -696,6 +706,21 @@ void AssetsManagerEx::startUpdate() } // Save current download manifest information for resuming _tempManifest->saveToFile(_tempManifestPath); + // Preprocessing local files in previous version and creating download folders + for (auto it = diff_map.begin(); it != diff_map.end(); ++it) + { + Manifest::AssetDiff diff = it->second; + if (diff.type != Manifest::DiffType::DELETED) + { + std::string path = diff.asset.path; + DownloadUnit unit; + unit.customId = it->first; + unit.srcUrl = packageUrl + path; + unit.storagePath = _tempStoragePath + path; + unit.size = diff.asset.size; + _downloadUnits.emplace(unit.customId, unit); + } + } _totalWaitToDownload = _totalToDownload = (int)_downloadUnits.size(); this->batchDownload(); @@ -710,17 +735,44 @@ void AssetsManagerEx::updateSucceed() { // Every thing is correctly downloaded, do the following // 1. rename temporary manifest to valid manifest - _fileUtils->renameFile(_storagePath, TEMP_MANIFEST_FILENAME, MANIFEST_FILENAME); - // 2. swap the localManifest - if (_localManifest != nullptr) - _localManifest->release(); + _fileUtils->renameFile(_tempStoragePath, TEMP_MANIFEST_FILENAME, MANIFEST_FILENAME); + // 2. merge temporary storage path to storage path so that temporary version turns to cached version + if (_fileUtils->isDirectoryExist(_tempStoragePath)) + { + // Merging all files in temp storage path to storage path + std::vector files; + _fileUtils->listFilesRecursively(_tempStoragePath, &files); + int baseOffset = (int)_tempStoragePath.length(); + std::string relativePath, dstPath; + for (std::vector::iterator it = files.begin(); it != files.end(); ++it) + { + relativePath.assign((*it).substr(baseOffset)); + dstPath.assign(_storagePath + relativePath); + // Create directory + if (relativePath.back() == '/') + { + _fileUtils->createDirectory(dstPath); + } + // Copy file + else + { + _fileUtils->removeFile(dstPath); + _fileUtils->renameFile(*it, dstPath); + } + } + // Remove temp storage path + _fileUtils->removeDirectory(_tempStoragePath); + } + // 3. swap the localManifest + CC_SAFE_RELEASE(_localManifest); _localManifest = _remoteManifest; + _localManifest->setManifestRoot(_storagePath); _remoteManifest = nullptr; - // 3. make local manifest take effect + // 4. make local manifest take effect prepareLocalManifest(); - // 4. Set update state + // 5. Set update state _updateState = State::UP_TO_DATE; - // 5. Notify finished event + // 6. Notify finished event dispatchUpdateEvent(EventAssetsManagerEx::EventCode::UPDATE_FINISHED); } @@ -864,7 +916,7 @@ void AssetsManagerEx::updateAssets(const DownloadUnits& assets) } else if (_totalToDownload == 0) { - updateSucceed(); + onDownloadUnitsFinished(); } } } @@ -1053,8 +1105,8 @@ void AssetsManagerEx::onSuccess(const std::string &/*srcUrl*/, const std::string void AssetsManagerEx::destroyDownloadedVersion() { - _fileUtils->removeFile(_cacheVersionPath); - _fileUtils->removeFile(_cacheManifestPath); + _fileUtils->removeDirectory(_storagePath); + _fileUtils->removeDirectory(_tempStoragePath); } void AssetsManagerEx::batchDownload() @@ -1097,6 +1149,7 @@ void AssetsManagerEx::queueDowload() _currConcurrentTask++; DownloadUnit& unit = _downloadUnits[key]; + _fileUtils->createDirectory(basename(unit.storagePath)); _downloader->createDownloadFileTask(unit.srcUrl, unit.storagePath, unit.customId); _tempManifest->setAssetDownloadState(key, Manifest::DownloadState::DOWNLOADING); diff --git a/native/extensions/assets-manager/AssetsManagerEx.h b/native/extensions/assets-manager/AssetsManagerEx.h index a9b5e6e3d6..90605047ea 100644 --- a/native/extensions/assets-manager/AssetsManagerEx.h +++ b/native/extensions/assets-manager/AssetsManagerEx.h @@ -236,11 +236,14 @@ private: //! The reference to the local assets const std::unordered_map *_assets; - //! The path to store downloaded resources. + //! The path to store successfully downloaded version. std::string _storagePath; - //! The local path of cached version file - std::string _cacheVersionPath; + //! The path to store downloading version. + std::string _tempStoragePath; + + //! The local path of cached temporary version file + std::string _tempVersionPath; //! The local path of cached manifest file std::string _cacheManifestPath; diff --git a/native/extensions/assets-manager/Manifest.cpp b/native/extensions/assets-manager/Manifest.cpp index 9696da83f8..12e0ec0ab2 100644 --- a/native/extensions/assets-manager/Manifest.cpp +++ b/native/extensions/assets-manager/Manifest.cpp @@ -266,20 +266,6 @@ void Manifest::genResumeAssetsList(DownloadUnits *units) const } } -void Manifest::cleanupDownloadingAssets() const -{ - for (auto it = _assets.begin(); it != _assets.end(); ++it) - { - Asset asset = it->second; - - if (asset.downloadState == DownloadState::DOWNLOADING) - { - std::string storagePath = _manifestRoot + asset.path; - _fileUtils->removeFile(storagePath); - } - } -} - std::vector Manifest::getSearchPaths() const { std::vector searchPaths; diff --git a/native/extensions/assets-manager/Manifest.h b/native/extensions/assets-manager/Manifest.h index b77f5168f9..f2eeff59f4 100644 --- a/native/extensions/assets-manager/Manifest.h +++ b/native/extensions/assets-manager/Manifest.h @@ -157,10 +157,6 @@ protected: */ void genResumeAssetsList(DownloadUnits *units) const; - /** @brief Cleanup assets that is in downloading state for resuming - */ - void cleanupDownloadingAssets() const; - /** @brief Prepend all search paths to the FileUtils. */ void prependSearchPaths(); @@ -200,6 +196,8 @@ protected: */ void setAssetDownloadState(const std::string &key, const DownloadState &state); + void setManifestRoot(const std::string &root) {_manifestRoot = root;}; + private: //! Indicate whether the version informations have been fully loaded From 3d02d7e5a6c54fc15319d7b5df301a51631cea7d Mon Sep 17 00:00:00 2001 From: pandamicro Date: Fri, 13 Jan 2017 10:31:56 +0800 Subject: [PATCH 6/6] Fix action and custom event listener memory management issue --- native/cocos/2d/CCActionManager.cpp | 16 ++++++++++++++-- .../manual/jsb_event_dispatcher_manual.cpp | 9 +-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/native/cocos/2d/CCActionManager.cpp b/native/cocos/2d/CCActionManager.cpp index 42e3bb1759..e9d47d7c02 100644 --- a/native/cocos/2d/CCActionManager.cpp +++ b/native/cocos/2d/CCActionManager.cpp @@ -239,6 +239,8 @@ void ActionManager::removeAllActionsFromTarget(Node *target) element->currentActionSalvaged = true; } + ccArrayRemoveAllObjects(element->actions); + #if CC_ENABLE_GC_FOR_NATIVE_OBJECTS auto sEngine = ScriptEngineManager::getInstance()->getScriptEngine(); if (sEngine) @@ -247,11 +249,14 @@ void ActionManager::removeAllActionsFromTarget(Node *target) for (int i = 0; i < limit; ++i) { Action *action = (Action*)element->actions->arr[i]; - sEngine->releaseScriptObject(this, action); + if (element->currentActionSalvaged && element->currentAction == action) + { + sEngine->releaseScriptObject(this, action); + } } } #endif // CC_ENABLE_GC_FOR_NATIVE_OBJECTS - ccArrayRemoveAllObjects(element->actions); + if (_currentTarget == element) { _currentTargetSalvaged = true; @@ -454,6 +459,13 @@ void ActionManager::update(float dt) // accidentally deallocating itself before finishing its step, we retained // it. Now that step is done, it's safe to release it. _currentTarget->currentAction->release(); +#if CC_ENABLE_GC_FOR_NATIVE_OBJECTS + auto sEngine = ScriptEngineManager::getInstance()->getScriptEngine(); + if (sEngine) + { + sEngine->releaseScriptObject(this, _currentTarget->currentAction); + } +#endif // CC_ENABLE_GC_FOR_NATIVE_OBJECTS } else if (_currentTarget->currentAction->isDone()) { diff --git a/native/cocos/scripting/js-bindings/manual/jsb_event_dispatcher_manual.cpp b/native/cocos/scripting/js-bindings/manual/jsb_event_dispatcher_manual.cpp index 3f580f32db..8434745f0d 100644 --- a/native/cocos/scripting/js-bindings/manual/jsb_event_dispatcher_manual.cpp +++ b/native/cocos/scripting/js-bindings/manual/jsb_event_dispatcher_manual.cpp @@ -304,14 +304,12 @@ bool js_EventDispatcher_addCustomEventListener(JSContext *cx, uint32_t argc, jsv if (argc == 2) { std::string arg0; std::function arg1; - JSFunctionWrapper *wrapper = nullptr; ok &= jsval_to_std_string(cx, args.get(0), &arg0); do { if(JS_TypeOfValue(cx, args.get(1)) == JSTYPE_FUNCTION) { JS::RootedObject jstarget(cx, args.thisv().toObjectOrNull()); - std::shared_ptr func(new JSFunctionWrapper(cx, jstarget, args.get(1))); - wrapper = func.get(); + std::shared_ptr func(new JSFunctionWrapper(cx, jstarget, args.get(1), args.thisv())); auto lambda = [=](cocos2d::EventCustom* event) -> void { JSB_AUTOCOMPARTMENT_WITH_GLOBAL_OBJCET jsval largv[1]; @@ -340,11 +338,6 @@ bool js_EventDispatcher_addCustomEventListener(JSContext *cx, uint32_t argc, jsv JS::RootedValue jsret(cx); if (ret) { jsret = OBJECT_TO_JSVAL(js_get_or_create_jsobject(cx, ret)); - ScriptingCore::getInstance()->retainScriptObject(cobj, ret); - if (wrapper) - { - wrapper->setOwner(cx, jsret); - } args.rval().set(jsret); } else { jsret = JSVAL_NULL;