From 64b9b4c771430dcb0d65d557402b5f8bf81aa5a0 Mon Sep 17 00:00:00 2001 From: andreykproductengine Date: Thu, 1 Feb 2018 16:35:02 +0200 Subject: MAINT-2338 Implemented binary parser depth control --- indra/llcommon/llsdserialize.cpp | 43 ++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 17 deletions(-) (limited to 'indra/llcommon/llsdserialize.cpp') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index be54ed053b..580e87954b 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -51,6 +51,7 @@ // File constants static const int MAX_HDR_LEN = 20; +static const S32 UNZIP_LLSD_MAX_DEPTH = 96; static const char LEGACY_NON_HEADER[] = ""; const std::string LLSD_BINARY_HEADER("LLSD/Binary"); const std::string LLSD_XML_HEADER("LLSD/XML"); @@ -317,11 +318,11 @@ LLSDParser::LLSDParser() LLSDParser::~LLSDParser() { } -S32 LLSDParser::parse(std::istream& istr, LLSD& data, S32 max_bytes) +S32 LLSDParser::parse(std::istream& istr, LLSD& data, S32 max_bytes, S32 max_depth) { mCheckLimits = (LLSDSerialize::SIZE_UNLIMITED == max_bytes) ? false : true; mMaxBytesLeft = max_bytes; - return doParse(istr, data); + return doParse(istr, data, max_depth); } @@ -403,7 +404,7 @@ LLSDNotationParser::~LLSDNotationParser() { } // virtual -S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const +S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) const { // map: { string:object, string:object } // array: [ object, object, object ] @@ -418,6 +419,10 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const // binary: b##"ff3120ab1" | b(size)"raw data" char c; c = istr.peek(); + if (max_depth == 0) + { + return PARSE_FAILURE; + } while(isspace(c)) { // pop the whitespace. @@ -434,7 +439,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const { case '{': { - S32 child_count = parseMap(istr, data); + S32 child_count = parseMap(istr, data, max_depth - 1); if((child_count == PARSE_FAILURE) || data.isUndefined()) { parse_count = PARSE_FAILURE; @@ -453,7 +458,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const case '[': { - S32 child_count = parseArray(istr, data); + S32 child_count = parseArray(istr, data, max_depth - 1); if((child_count == PARSE_FAILURE) || data.isUndefined()) { parse_count = PARSE_FAILURE; @@ -658,7 +663,7 @@ S32 LLSDNotationParser::doParse(std::istream& istr, LLSD& data) const return parse_count; } -S32 LLSDNotationParser::parseMap(std::istream& istr, LLSD& map) const +S32 LLSDNotationParser::parseMap(std::istream& istr, LLSD& map, S32 max_depth) const { // map: { string:object, string:object } map = LLSD::emptyMap(); @@ -693,7 +698,7 @@ S32 LLSDNotationParser::parseMap(std::istream& istr, LLSD& map) const } putback(istr, c); LLSD child; - S32 count = doParse(istr, child); + S32 count = doParse(istr, child, max_depth); if(count > 0) { // There must be a value for every key, thus @@ -718,7 +723,7 @@ S32 LLSDNotationParser::parseMap(std::istream& istr, LLSD& map) const return parse_count; } -S32 LLSDNotationParser::parseArray(std::istream& istr, LLSD& array) const +S32 LLSDNotationParser::parseArray(std::istream& istr, LLSD& array, S32 max_depth) const { // array: [ object, object, object ] array = LLSD::emptyArray(); @@ -737,7 +742,7 @@ S32 LLSDNotationParser::parseArray(std::istream& istr, LLSD& array) const continue; } putback(istr, c); - S32 count = doParse(istr, child); + S32 count = doParse(istr, child, max_depth); if(PARSE_FAILURE == count) { return PARSE_FAILURE; @@ -869,7 +874,7 @@ LLSDBinaryParser::~LLSDBinaryParser() } // virtual -S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const +S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data, S32 max_depth) const { /** * Undefined: '!'
@@ -893,12 +898,16 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const { return 0; } + if (max_depth == 0) + { + return PARSE_FAILURE; + } S32 parse_count = 1; switch(c) { case '{': { - S32 child_count = parseMap(istr, data); + S32 child_count = parseMap(istr, data, max_depth - 1); if((child_count == PARSE_FAILURE) || data.isUndefined()) { parse_count = PARSE_FAILURE; @@ -917,7 +926,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const case '[': { - S32 child_count = parseArray(istr, data); + S32 child_count = parseArray(istr, data, max_depth - 1); if((child_count == PARSE_FAILURE) || data.isUndefined()) { parse_count = PARSE_FAILURE; @@ -1098,7 +1107,7 @@ S32 LLSDBinaryParser::doParse(std::istream& istr, LLSD& data) const return parse_count; } -S32 LLSDBinaryParser::parseMap(std::istream& istr, LLSD& map) const +S32 LLSDBinaryParser::parseMap(std::istream& istr, LLSD& map, S32 max_depth) const { map = LLSD::emptyMap(); U32 value_nbo = 0; @@ -1128,7 +1137,7 @@ S32 LLSDBinaryParser::parseMap(std::istream& istr, LLSD& map) const } } LLSD child; - S32 child_count = doParse(istr, child); + S32 child_count = doParse(istr, child, max_depth); if(child_count > 0) { // There must be a value for every key, thus child_count @@ -1152,7 +1161,7 @@ S32 LLSDBinaryParser::parseMap(std::istream& istr, LLSD& map) const return parse_count; } -S32 LLSDBinaryParser::parseArray(std::istream& istr, LLSD& array) const +S32 LLSDBinaryParser::parseArray(std::istream& istr, LLSD& array, S32 max_depth) const { array = LLSD::emptyArray(); U32 value_nbo = 0; @@ -1168,7 +1177,7 @@ S32 LLSDBinaryParser::parseArray(std::istream& istr, LLSD& array) const while((c != ']') && (count < size) && istr.good()) { LLSD child; - S32 child_count = doParse(istr, child); + S32 child_count = doParse(istr, child, max_depth); if(PARSE_FAILURE == child_count) { return PARSE_FAILURE; @@ -2238,7 +2247,7 @@ LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, std::istream& is, return ZR_MEM_ERROR; } - if (!LLSDSerialize::fromBinary(data, istr, cur_size)) + if (!LLSDSerialize::fromBinary(data, istr, cur_size, UNZIP_LLSD_MAX_DEPTH)) { free(result); return ZR_PARSE_ERROR; -- cgit v1.3 From f22c1bcf2014346211ecc32c21d48cc88aca8e07 Mon Sep 17 00:00:00 2001 From: andreykproductengine Date: Mon, 19 Feb 2018 21:30:48 +0200 Subject: MAINT-8022 Handling memory errors in unzip_llsdNavMesh --- indra/llcommon/llsdserialize.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'indra/llcommon/llsdserialize.cpp') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 580e87954b..26a4967659 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -2262,13 +2262,24 @@ LLUZipHelper::EZipRresult LLUZipHelper::unzip_llsd(LLSD& data, std::istream& is, //and trailers are different for the formats. U8* unzip_llsdNavMesh( bool& valid, unsigned int& outsize, std::istream& is, S32 size ) { + if (size == 0) + { + LL_WARNS() << "No data to unzip." << LL_ENDL; + return NULL; + } + U8* result = NULL; U32 cur_size = 0; z_stream strm; const U32 CHUNK = 0x4000; - U8 *in = new U8[size]; + U8 *in = new(std::nothrow) U8[size]; + if (in == NULL) + { + LL_WARNS() << "Memory allocation failure." << LL_ENDL; + return NULL; + } is.read((char*) in, size); U8 out[CHUNK]; -- cgit v1.3 From e32b4f481a3da183607cfd02f873e41e2a77214e Mon Sep 17 00:00:00 2001 From: andreykproductengine Date: Mon, 19 Feb 2018 21:01:34 +0200 Subject: MAINT-8308 Additional logging for mesh processing --- indra/llcommon/llsdserialize.cpp | 5 +++- indra/newview/llmeshrepository.cpp | 55 ++++++++++++++++++++++++++------------ indra/newview/llmeshrepository.h | 11 +++++++- 3 files changed, 52 insertions(+), 19 deletions(-) (limited to 'indra/llcommon/llsdserialize.cpp') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index 26a4967659..1aaff5628f 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -2323,7 +2323,10 @@ U8* unzip_llsdNavMesh( bool& valid, unsigned int& outsize, std::istream& is, S32 U8* new_result = (U8*) realloc(result, cur_size + have); if (new_result == NULL) { - LL_WARNS() << "Failed to unzip LLSD NavMesh block: can't reallocate memory, current size: " << cur_size << " bytes; requested " << cur_size + have << " bytes." << LL_ENDL; + LL_WARNS() << "Failed to unzip LLSD NavMesh block: can't reallocate memory, current size: " << cur_size + << " bytes; requested " << cur_size + have + << " bytes; total syze: ." << size << " bytes." + << LL_ENDL; inflateEnd(&strm); if (result) { diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index fdaa28b22b..69e69b134a 100644 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -1618,7 +1618,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod) if (!zero) { //attempt to parse - if (lodReceived(mesh_params, lod, buffer, size)) + if (lodReceived(mesh_params, lod, buffer, size) == MESH_OK) { delete[] buffer; return true; @@ -1734,11 +1734,11 @@ bool LLMeshRepoThread::headerReceived(const LLVolumeParams& mesh_params, U8* dat return true; } -bool LLMeshRepoThread::lodReceived(const LLVolumeParams& mesh_params, S32 lod, U8* data, S32 data_size) +EMeshProcessingResult LLMeshRepoThread::lodReceived(const LLVolumeParams& mesh_params, S32 lod, U8* data, S32 data_size) { if (data == NULL || data_size == 0) { - return false; + return MESH_NO_DATA; } LLPointer volume = new LLVolume(mesh_params, LLVolumeLODGroup::getVolumeScaleFromDetail(lod)); @@ -1751,7 +1751,7 @@ bool LLMeshRepoThread::lodReceived(const LLVolumeParams& mesh_params, S32 lod, U catch (std::bad_alloc) { // out of memory, we won't be able to process this mesh - return false; + return MESH_OUT_OF_MEMORY; } if (volume->unpackVolumeFaces(stream, data_size)) @@ -1763,11 +1763,11 @@ bool LLMeshRepoThread::lodReceived(const LLVolumeParams& mesh_params, S32 lod, U LLMutexLock lock(mMutex); mLoadedQ.push(mesh); } - return true; + return MESH_OK; } } - return false; + return MESH_UNKNOWN; } bool LLMeshRepoThread::skinInfoReceived(const LLUUID& mesh_id, U8* data, S32 data_size) @@ -2952,6 +2952,11 @@ void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRespo body->read(body_offset, (char *) data, data_size - body_offset); LLMeshRepository::sBytesReceived += data_size; } + else + { + LL_WARNS(LOG_MESH) << "Failed to allocate " << data_size - body_offset << " memory for mesh response" << LL_ENDL; + processFailure(LLCore::HttpStatus(LLCore::HttpStatus::LLCORE, LLCore::HE_BAD_ALLOC)); + } } processData(body, body_offset, data, data_size - body_offset); @@ -3127,27 +3132,43 @@ void LLMeshLODHandler::processData(LLCore::BufferArray * /* body */, S32 /* body U8 * data, S32 data_size) { if ((!MESH_LOD_PROCESS_FAILED) - && ((data != NULL) == (data_size > 0)) // if we have data but no size or have size but no data, something is wrong - && gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size)) + && ((data != NULL) == (data_size > 0))) // if we have data but no size or have size but no data, something is wrong { - // good fetch from sim, write to VFS for caching - LLVFile file(gVFS, mMeshParams.getSculptID(), LLAssetType::AT_MESH, LLVFile::WRITE); + EMeshProcessingResult result = gMeshRepo.mThread->lodReceived(mMeshParams, mLOD, data, data_size); + if (result == MESH_OK) + { + // good fetch from sim, write to VFS for caching + LLVFile file(gVFS, mMeshParams.getSculptID(), LLAssetType::AT_MESH, LLVFile::WRITE); - S32 offset = mOffset; - S32 size = mRequestedBytes; + S32 offset = mOffset; + S32 size = mRequestedBytes; - if (file.getSize() >= offset+size) + if (file.getSize() >= offset+size) + { + file.seek(offset); + file.write(data, size); + LLMeshRepository::sCacheBytesWritten += size; + ++LLMeshRepository::sCacheWrites; + } + } + else { - file.seek(offset); - file.write(data, size); - LLMeshRepository::sCacheBytesWritten += size; - ++LLMeshRepository::sCacheWrites; + LL_WARNS(LOG_MESH) << "Error during mesh LOD processing. ID: " << mMeshParams.getSculptID() + << ", Reason: " << result + << " LOD: " << mLOD + << " Data size: " << data_size + << " Not retrying." + << LL_ENDL; + LLMutexLock lock(gMeshRepo.mThread->mMutex); + gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, mLOD)); } } else { LL_WARNS(LOG_MESH) << "Error during mesh LOD processing. ID: " << mMeshParams.getSculptID() << ", Unknown reason. Not retrying." + << " LOD: " << mLOD + << " Data size: " << data_size << LL_ENDL; LLMutexLock lock(gMeshRepo.mThread->mMutex); gMeshRepo.mThread->mUnavailableQ.push(LLMeshRepoThread::LODRequest(mMeshParams, mLOD)); diff --git a/indra/newview/llmeshrepository.h b/indra/newview/llmeshrepository.h index 23af837f6f..e07a00bf03 100644 --- a/indra/newview/llmeshrepository.h +++ b/indra/newview/llmeshrepository.h @@ -51,6 +51,15 @@ class LLCondition; class LLVFS; class LLMeshRepository; +typedef enum e_mesh_processing_result_enum +{ + MESH_OK = 0, + MESH_NO_DATA = 1, + MESH_OUT_OF_MEMORY, + MESH_HTTP_REQUEST_FAILED, + MESH_UNKNOWN +} EMeshProcessingResult; + class LLMeshUploadData { public: @@ -298,7 +307,7 @@ public: bool fetchMeshHeader(const LLVolumeParams& mesh_params); bool fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod); bool headerReceived(const LLVolumeParams& mesh_params, U8* data, S32 data_size); - bool lodReceived(const LLVolumeParams& mesh_params, S32 lod, U8* data, S32 data_size); + EMeshProcessingResult lodReceived(const LLVolumeParams& mesh_params, S32 lod, U8* data, S32 data_size); bool skinInfoReceived(const LLUUID& mesh_id, U8* data, S32 data_size); bool decompositionReceived(const LLUUID& mesh_id, U8* data, S32 data_size); bool physicsShapeReceived(const LLUUID& mesh_id, U8* data, S32 data_size); -- cgit v1.3