From 18483e902c0c821dc5d0bb8349f83d1a3b34d5a8 Mon Sep 17 00:00:00 2001 From: andreykproductengine Date: Mon, 18 Sep 2017 18:11:04 +0300 Subject: MAINT-7779 Protections against wrong data --- indra/newview/lltexturecache.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'indra/newview/lltexturecache.cpp') diff --git a/indra/newview/lltexturecache.cpp b/indra/newview/lltexturecache.cpp index f0c28041d1..87a2c0ae60 100644 --- a/indra/newview/lltexturecache.cpp +++ b/indra/newview/lltexturecache.cpp @@ -550,9 +550,11 @@ bool LLTextureCacheRemoteWorker::doWrite() { if ((mOffset != 0) // We currently do not support write offsets || (mDataSize <= 0) // Things will go badly wrong if mDataSize is nul or negative... - || (mImageSize < mDataSize)) + || (mImageSize < mDataSize) + || (mRawDiscardLevel < 0) + || (mRawImage->isBufferInvalid())) // decode failed or malfunctioned, don't write { - LL_WARNS() << "INIT state check failed" << LL_ENDL; + LL_WARNS() << "INIT state check failed for image: " << mID << " Size: " << mImageSize << " DataSize: " << mDataSize << " Discard:" << mRawDiscardLevel << LL_ENDL; mDataSize = -1; // failed done = true; } @@ -577,15 +579,12 @@ bool LLTextureCacheRemoteWorker::doWrite() idx = mCache->setHeaderCacheEntry(mID, entry, mImageSize, mDataSize); // create the new entry. if(idx >= 0) { - // (almost always) write to the fast cache. - if (mRawImage->getDataSize()) + // write to the fast cache. + if(!mCache->writeToFastCache(idx, mRawImage, mRawDiscardLevel)) { - if(!mCache->writeToFastCache(idx, mRawImage, mRawDiscardLevel)) - { - LL_WARNS() << "writeToFastCache failed" << LL_ENDL; - mDataSize = -1; // failed - done = true; - } + LL_WARNS() << "writeToFastCache failed" << LL_ENDL; + mDataSize = -1; // failed + done = true; } } } -- cgit v1.2.3 From 5844a9372fdd6e009d4eee0702204f4642f17bf8 Mon Sep 17 00:00:00 2001 From: andreykproductengine Date: Thu, 28 Sep 2017 17:10:38 +0300 Subject: SL-775 Texture cache purge improvements --- indra/newview/lltexturecache.cpp | 95 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 5 deletions(-) (limited to 'indra/newview/lltexturecache.cpp') diff --git a/indra/newview/lltexturecache.cpp b/indra/newview/lltexturecache.cpp index 87a2c0ae60..1aa95f996a 100644 --- a/indra/newview/lltexturecache.cpp +++ b/indra/newview/lltexturecache.cpp @@ -53,6 +53,7 @@ const F32 TEXTURE_CACHE_PURGE_AMOUNT = .20f; // % amount to reduce the cache by const F32 TEXTURE_CACHE_LRU_SIZE = .10f; // % amount for LRU list (low overhead to regenerate) const S32 TEXTURE_FAST_CACHE_ENTRY_OVERHEAD = sizeof(S32) * 4; //w, h, c, level const S32 TEXTURE_FAST_CACHE_ENTRY_SIZE = 16 * 16 * 4 + TEXTURE_FAST_CACHE_ENTRY_OVERHEAD; +const F32 TEXTURE_LAZY_PURGE_TIME_LIMIT = .01f; // 10ms class LLTextureCacheWorker : public LLWorkerClass { @@ -1634,6 +1635,91 @@ void LLTextureCache::purgeAllTextures(bool purge_directories) LL_INFOS() << "The entire texture cache is cleared." << LL_ENDL ; } +void LLTextureCache::purgeTexturesLazy(F32 time_limit) +{ + if (mReadOnly) + { + return; + } + + if (!mThreaded) + { + // *FIX:Mani - watchdog off. + LLAppViewer::instance()->pauseMainloopTimeout(); + } + + // time_limit doesn't account for lock time + LLMutexLock lock(&mHeaderMutex); + + if (mPurgeEntryList.empty()) + { + // Read the entries list and form list of textures to purge + std::vector entries; + U32 num_entries = openAndReadEntries(entries); + if (!num_entries) + { + return; // nothing to purge + } + + // Use mTexturesSizeMap to collect UUIDs of textures with bodies + typedef std::set > time_idx_set_t; + std::set > time_idx_set; + for (size_map_t::iterator iter1 = mTexturesSizeMap.begin(); + iter1 != mTexturesSizeMap.end(); ++iter1) + { + if (iter1->second > 0) + { + id_map_t::iterator iter2 = mHeaderIDMap.find(iter1->first); + if (iter2 != mHeaderIDMap.end()) + { + S32 idx = iter2->second; + time_idx_set.insert(std::make_pair(entries[idx].mTime, idx)); + } + else + { + LL_ERRS() << "mTexturesSizeMap / mHeaderIDMap corrupted." << LL_ENDL; + } + } + } + + S64 cache_size = mTexturesSizeTotal; + S64 purged_cache_size = (sCacheMaxTexturesSize * (S64)((1.f - TEXTURE_CACHE_PURGE_AMOUNT) * 100)) / 100; + for (time_idx_set_t::iterator iter = time_idx_set.begin(); + iter != time_idx_set.end(); ++iter) + { + S32 idx = iter->second; + if (cache_size >= purged_cache_size) + { + cache_size -= entries[idx].mBodySize; + mPurgeEntryList.push_back(std::pair(idx, entries[idx])); + } + else + { + break; + } + } + LL_DEBUGS() << "Formed Purge list of " << mPurgeEntryList.size() << " entries" << LL_ENDL; + } + else + { + // Remove collected entried + LLTimer timer; + while (!mPurgeEntryList.empty() && timer.getElapsedTimeF32() < time_limit) + { + S32 idx = mPurgeEntryList.back().first; + Entry entry = mPurgeEntryList.back().second; + mPurgeEntryList.pop_back(); + // make sure record is still valid + id_map_t::iterator iter_header = mHeaderIDMap.find(entry.mID); + if (iter_header != mHeaderIDMap.end() && iter_header->second == idx) + { + removeEntry(idx, entry, getTextureFileName(entry.mID)); + writeEntryToHeaderImmediately(idx, entry); + } + } + } +} + void LLTextureCache::purgeTextures(bool validate) { if (mReadOnly) @@ -1887,11 +1973,10 @@ LLTextureCache::handle_t LLTextureCache::writeToCache(const LLUUID& id, U32 prio } if (mDoPurge) { - // NOTE: This may cause an occasional hiccup, - // but it really needs to be done on the control thread - // (i.e. here) - purgeTextures(false); - mDoPurge = FALSE; + // NOTE: Needs to be done on the control thread + // (i.e. here) + purgeTexturesLazy(TEXTURE_LAZY_PURGE_TIME_LIMIT); + mDoPurge = !mPurgeEntryList.empty(); } LLMutexLock lock(&mWorkersMutex); LLTextureCacheWorker* worker = new LLTextureCacheRemoteWorker(this, priority, id, -- cgit v1.2.3 From e0265d3754b190d28d5a5a55fc1a1e510d9aef59 Mon Sep 17 00:00:00 2001 From: Mnikolenko Productengine Date: Tue, 3 Oct 2017 12:00:11 +0300 Subject: build fix --- indra/newview/lltexturecache.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'indra/newview/lltexturecache.cpp') diff --git a/indra/newview/lltexturecache.cpp b/indra/newview/lltexturecache.cpp index 1aa95f996a..1000377718 100644 --- a/indra/newview/lltexturecache.cpp +++ b/indra/newview/lltexturecache.cpp @@ -1713,7 +1713,8 @@ void LLTextureCache::purgeTexturesLazy(F32 time_limit) id_map_t::iterator iter_header = mHeaderIDMap.find(entry.mID); if (iter_header != mHeaderIDMap.end() && iter_header->second == idx) { - removeEntry(idx, entry, getTextureFileName(entry.mID)); + std::string tex_filename = getTextureFileName(entry.mID); + removeEntry(idx, entry, tex_filename); writeEntryToHeaderImmediately(idx, entry); } } -- cgit v1.2.3 From 5ee30b7da5ccd92baa1099a579fda34dab9e3d49 Mon Sep 17 00:00:00 2001 From: andreykproductengine Date: Wed, 11 Oct 2017 19:37:35 +0300 Subject: MAINT-7895 Fix for setHeaderCacheEntry infinite loop --- indra/newview/lltexturecache.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'indra/newview/lltexturecache.cpp') diff --git a/indra/newview/lltexturecache.cpp b/indra/newview/lltexturecache.cpp index 1000377718..71d3731f93 100644 --- a/indra/newview/lltexturecache.cpp +++ b/indra/newview/lltexturecache.cpp @@ -1879,24 +1879,27 @@ S32 LLTextureCache::getHeaderCacheEntry(const LLUUID& id, Entry& entry) S32 LLTextureCache::setHeaderCacheEntry(const LLUUID& id, Entry& entry, S32 imagesize, S32 datasize) { mHeaderMutex.lock(); - S32 idx = openAndReadEntry(id, entry, true); + S32 idx = openAndReadEntry(id, entry, true); // read or create mHeaderMutex.unlock(); - if (idx >= 0) - { - updateEntry(idx, entry, imagesize, datasize); - } - - if(idx < 0) // retry + if(idx < 0) // retry once { readHeaderCache(); // We couldn't write an entry, so refresh the LRU - + mHeaderMutex.lock(); - llassert_always(!mLRU.empty() || mHeaderEntriesInfo.mEntries < sCacheMaxEntries); + idx = openAndReadEntry(id, entry, true); mHeaderMutex.unlock(); + } - idx = setHeaderCacheEntry(id, entry, imagesize, datasize); // assert above ensures no inf. recursion + if (idx >= 0) + { + updateEntry(idx, entry, imagesize, datasize); } + else + { + LL_WARNS() << "Failed to set cache entry for image: " << id << LL_ENDL; + } + return idx; } -- cgit v1.2.3 From 7697f7f9a651766cd7b3e88824dd546e66e3499b Mon Sep 17 00:00:00 2001 From: andreykproductengine Date: Mon, 23 Oct 2017 17:15:00 +0300 Subject: MAINT-7929 Viewer shouldn't attempt to access unreadable cache indefientely --- indra/newview/lltexturecache.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'indra/newview/lltexturecache.cpp') diff --git a/indra/newview/lltexturecache.cpp b/indra/newview/lltexturecache.cpp index 71d3731f93..7c88cdaf9f 100644 --- a/indra/newview/lltexturecache.cpp +++ b/indra/newview/lltexturecache.cpp @@ -936,7 +936,7 @@ BOOL LLTextureCache::isInLocal(const LLUUID& id) ////////////////////////////////////////////////////////////////////////////// //static -F32 LLTextureCache::sHeaderCacheVersion = 1.7f; +F32 LLTextureCache::sHeaderCacheVersion = 1.71f; U32 LLTextureCache::sCacheMaxEntries = 1024 * 1024; //~1 million textures. S64 LLTextureCache::sCacheMaxTexturesSize = 0; // no limit std::string LLTextureCache::sHeaderCacheEncoderVersion = LLImageJ2C::getEngineInfo(); @@ -1898,6 +1898,9 @@ S32 LLTextureCache::setHeaderCacheEntry(const LLUUID& id, Entry& entry, S32 imag else { LL_WARNS() << "Failed to set cache entry for image: " << id << LL_ENDL; + // We couldn't write to file, switch to read only mode and clear data + setReadOnly(true); + clearCorruptedCache(); // won't remove files due to "read only" } return idx; @@ -2050,7 +2053,7 @@ LLPointer LLTextureCache::readFromFastCache(const LLUUID& id, S32& d bool LLTextureCache::writeToFastCache(S32 id, LLPointer raw, S32 discardlevel) { //rescale image if needed - if (raw.isNull() || !raw->getData()) + if (raw.isNull() || raw->isBufferInvalid() || !raw->getData()) { LL_ERRS() << "Attempted to write NULL raw image to fastcache" << LL_ENDL; return false; -- cgit v1.2.3 From 76c7eb73109189871044ff7b19235af33d205bd3 Mon Sep 17 00:00:00 2001 From: andreykproductengine Date: Mon, 23 Oct 2017 17:28:18 +0300 Subject: Backed out changeset: bfa432c3c7d5 (SL-775) --- indra/newview/lltexturecache.cpp | 96 +++------------------------------------- 1 file changed, 5 insertions(+), 91 deletions(-) (limited to 'indra/newview/lltexturecache.cpp') diff --git a/indra/newview/lltexturecache.cpp b/indra/newview/lltexturecache.cpp index 7c88cdaf9f..062ee37864 100644 --- a/indra/newview/lltexturecache.cpp +++ b/indra/newview/lltexturecache.cpp @@ -53,7 +53,6 @@ const F32 TEXTURE_CACHE_PURGE_AMOUNT = .20f; // % amount to reduce the cache by const F32 TEXTURE_CACHE_LRU_SIZE = .10f; // % amount for LRU list (low overhead to regenerate) const S32 TEXTURE_FAST_CACHE_ENTRY_OVERHEAD = sizeof(S32) * 4; //w, h, c, level const S32 TEXTURE_FAST_CACHE_ENTRY_SIZE = 16 * 16 * 4 + TEXTURE_FAST_CACHE_ENTRY_OVERHEAD; -const F32 TEXTURE_LAZY_PURGE_TIME_LIMIT = .01f; // 10ms class LLTextureCacheWorker : public LLWorkerClass { @@ -1635,92 +1634,6 @@ void LLTextureCache::purgeAllTextures(bool purge_directories) LL_INFOS() << "The entire texture cache is cleared." << LL_ENDL ; } -void LLTextureCache::purgeTexturesLazy(F32 time_limit) -{ - if (mReadOnly) - { - return; - } - - if (!mThreaded) - { - // *FIX:Mani - watchdog off. - LLAppViewer::instance()->pauseMainloopTimeout(); - } - - // time_limit doesn't account for lock time - LLMutexLock lock(&mHeaderMutex); - - if (mPurgeEntryList.empty()) - { - // Read the entries list and form list of textures to purge - std::vector entries; - U32 num_entries = openAndReadEntries(entries); - if (!num_entries) - { - return; // nothing to purge - } - - // Use mTexturesSizeMap to collect UUIDs of textures with bodies - typedef std::set > time_idx_set_t; - std::set > time_idx_set; - for (size_map_t::iterator iter1 = mTexturesSizeMap.begin(); - iter1 != mTexturesSizeMap.end(); ++iter1) - { - if (iter1->second > 0) - { - id_map_t::iterator iter2 = mHeaderIDMap.find(iter1->first); - if (iter2 != mHeaderIDMap.end()) - { - S32 idx = iter2->second; - time_idx_set.insert(std::make_pair(entries[idx].mTime, idx)); - } - else - { - LL_ERRS() << "mTexturesSizeMap / mHeaderIDMap corrupted." << LL_ENDL; - } - } - } - - S64 cache_size = mTexturesSizeTotal; - S64 purged_cache_size = (sCacheMaxTexturesSize * (S64)((1.f - TEXTURE_CACHE_PURGE_AMOUNT) * 100)) / 100; - for (time_idx_set_t::iterator iter = time_idx_set.begin(); - iter != time_idx_set.end(); ++iter) - { - S32 idx = iter->second; - if (cache_size >= purged_cache_size) - { - cache_size -= entries[idx].mBodySize; - mPurgeEntryList.push_back(std::pair(idx, entries[idx])); - } - else - { - break; - } - } - LL_DEBUGS() << "Formed Purge list of " << mPurgeEntryList.size() << " entries" << LL_ENDL; - } - else - { - // Remove collected entried - LLTimer timer; - while (!mPurgeEntryList.empty() && timer.getElapsedTimeF32() < time_limit) - { - S32 idx = mPurgeEntryList.back().first; - Entry entry = mPurgeEntryList.back().second; - mPurgeEntryList.pop_back(); - // make sure record is still valid - id_map_t::iterator iter_header = mHeaderIDMap.find(entry.mID); - if (iter_header != mHeaderIDMap.end() && iter_header->second == idx) - { - std::string tex_filename = getTextureFileName(entry.mID); - removeEntry(idx, entry, tex_filename); - writeEntryToHeaderImmediately(idx, entry); - } - } - } -} - void LLTextureCache::purgeTextures(bool validate) { if (mReadOnly) @@ -1980,10 +1893,11 @@ LLTextureCache::handle_t LLTextureCache::writeToCache(const LLUUID& id, U32 prio } if (mDoPurge) { - // NOTE: Needs to be done on the control thread - // (i.e. here) - purgeTexturesLazy(TEXTURE_LAZY_PURGE_TIME_LIMIT); - mDoPurge = !mPurgeEntryList.empty(); + // NOTE: This may cause an occasional hiccup, + // but it really needs to be done on the control thread + // (i.e. here) + purgeTextures(false); + mDoPurge = FALSE; } LLMutexLock lock(&mWorkersMutex); LLTextureCacheWorker* worker = new LLTextureCacheRemoteWorker(this, priority, id, -- cgit v1.2.3 From 7161619d6c0ee790c5741d0df028e428052e5fcc Mon Sep 17 00:00:00 2001 From: andreykproductengine Date: Mon, 6 Nov 2017 16:46:16 +0200 Subject: MAINT-7978 - Fixed allocation crash in doRead(), workers should never crash --- indra/newview/lltexturecache.cpp | 112 ++++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 44 deletions(-) (limited to 'indra/newview/lltexturecache.cpp') diff --git a/indra/newview/lltexturecache.cpp b/indra/newview/lltexturecache.cpp index 062ee37864..435d833345 100644 --- a/indra/newview/lltexturecache.cpp +++ b/indra/newview/lltexturecache.cpp @@ -387,22 +387,35 @@ bool LLTextureCacheRemoteWorker::doRead() } // Allocate read buffer mReadData = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), mDataSize); - S32 bytes_read = LLAPRFile::readEx(local_filename, - mReadData, mOffset, mDataSize, mCache->getLocalAPRFilePool()); - if (bytes_read != mDataSize) + + if (mReadData) { - LL_WARNS() << "Error reading file from local cache: " << local_filename - << " Bytes: " << mDataSize << " Offset: " << mOffset + S32 bytes_read = LLAPRFile::readEx( local_filename, + mReadData, + mOffset, + mDataSize, + mCache->getLocalAPRFilePool()); + + if (bytes_read != mDataSize) + { + LL_WARNS() << "Error reading file from local cache: " << local_filename + << " Bytes: " << mDataSize << " Offset: " << mOffset << " / " << mDataSize << LL_ENDL; - mDataSize = 0; - FREE_MEM(LLImageBase::getPrivatePool(), mReadData); - mReadData = NULL; + mDataSize = 0; + FREE_MEM(LLImageBase::getPrivatePool(), mReadData); + mReadData = NULL; + } + else + { + mImageSize = local_size; + mImageLocal = TRUE; + } } else { - //LL_INFOS() << "texture " << mID.asString() << " found in local_assets" << LL_ENDL; - mImageSize = local_size; - mImageLocal = TRUE; + LL_WARNS() << "Error allocating memory for cache: " << local_filename + << " of size: " << mDataSize << LL_ENDL; + mDataSize = 0; } // We're done... done = true; @@ -477,44 +490,55 @@ bool LLTextureCacheRemoteWorker::doRead() // Reserve the whole data buffer first U8* data = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), mDataSize); - - // Set the data file pointers taking the read offset into account. 2 cases: - if (mOffset < TEXTURE_CACHE_ENTRY_SIZE) + if (data) { - // Offset within the header record. That means we read something from the header cache. - // Note: most common case is (mOffset = 0), so this is the "normal" code path. - data_offset = TEXTURE_CACHE_ENTRY_SIZE - mOffset; // i.e. TEXTURE_CACHE_ENTRY_SIZE if mOffset nul (common case) - file_offset = 0; - file_size = mDataSize - data_offset; - // Copy the raw data we've been holding from the header cache into the new sized buffer - llassert_always(mReadData); - memcpy(data, mReadData, data_offset); - FREE_MEM(LLImageBase::getPrivatePool(), mReadData); - mReadData = NULL; - } - else - { - // Offset bigger than the header record. That means we haven't read anything yet. - data_offset = 0; - file_offset = mOffset - TEXTURE_CACHE_ENTRY_SIZE; - file_size = mDataSize; - // No data from header cache to copy in that case, we skipped it all - } + // Set the data file pointers taking the read offset into account. 2 cases: + if (mOffset < TEXTURE_CACHE_ENTRY_SIZE) + { + // Offset within the header record. That means we read something from the header cache. + // Note: most common case is (mOffset = 0), so this is the "normal" code path. + data_offset = TEXTURE_CACHE_ENTRY_SIZE - mOffset; // i.e. TEXTURE_CACHE_ENTRY_SIZE if mOffset nul (common case) + file_offset = 0; + file_size = mDataSize - data_offset; + // Copy the raw data we've been holding from the header cache into the new sized buffer + llassert_always(mReadData); + memcpy(data, mReadData, data_offset); + FREE_MEM(LLImageBase::getPrivatePool(), mReadData); + mReadData = NULL; + } + else + { + // Offset bigger than the header record. That means we haven't read anything yet. + data_offset = 0; + file_offset = mOffset - TEXTURE_CACHE_ENTRY_SIZE; + file_size = mDataSize; + // No data from header cache to copy in that case, we skipped it all + } - // Now use that buffer as the object read buffer - llassert_always(mReadData == NULL); - mReadData = data; + // Now use that buffer as the object read buffer + llassert_always(mReadData == NULL); + mReadData = data; - // Read the data at last - S32 bytes_read = LLAPRFile::readEx(filename, - mReadData + data_offset, - file_offset, file_size, - mCache->getLocalAPRFilePool()); - if (bytes_read != file_size) + // Read the data at last + S32 bytes_read = LLAPRFile::readEx(filename, + mReadData + data_offset, + file_offset, file_size, + mCache->getLocalAPRFilePool()); + if (bytes_read != file_size) + { + LL_WARNS() << "LLTextureCacheWorker: " << mID + << " incorrect number of bytes read from body: " << bytes_read + << " / " << file_size << LL_ENDL; + FREE_MEM(LLImageBase::getPrivatePool(), mReadData); + mReadData = NULL; + mDataSize = -1; // failed + done = true; + } + } + else { LL_WARNS() << "LLTextureCacheWorker: " << mID - << " incorrect number of bytes read from body: " << bytes_read - << " / " << file_size << LL_ENDL; + << " failed to allocate memory for reading: " << mDataSize << LL_ENDL; FREE_MEM(LLImageBase::getPrivatePool(), mReadData); mReadData = NULL; mDataSize = -1; // failed -- cgit v1.2.3