From d9448c6f52218146113d1d5c5ca4c4d5f01dc5cf Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Thu, 17 Sep 2020 09:45:06 -0700 Subject: The folder where the disk cache lives was originally renamed from llvfs to llcache but @henri's suggestion that that doesn't reflect the other files in the same place and it should be llfilesystem is a good one so I changed it over --- indra/llfilesystem/lldiskcache.h | 82 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 indra/llfilesystem/lldiskcache.h (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h new file mode 100644 index 0000000000..7ad06a8689 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.h @@ -0,0 +1,82 @@ +/** + * @file lldiskcacke.h + * @brief Definition of virtual file + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLDISKCACHE_H +#define LL_LLDISKCACHE_H + +#include "lluuid.h" +#include "llassettype.h" + +class LLDiskCache +{ +public: + LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLDiskCache::READ); + ~LLDiskCache(); + + BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ + BOOL isReadComplete(); + S32 getLastBytesRead(); + BOOL eof(); + + BOOL write(const U8 *buffer, S32 bytes); + static BOOL writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type); + BOOL seek(S32 offset, S32 origin = -1); + S32 tell() const; + + S32 getSize(); + S32 getMaxSize(); + BOOL setMaxSize(S32 size); + BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); + BOOL remove(); + + bool isLocked(); + void waitForLock(); + + static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); + static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); + static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, + const LLUUID &new_file_id, const LLAssetType::EType new_file_type); + static S32 getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type); + + static void initClass(); + static void cleanupClass(); + +public: + static const S32 READ; + static const S32 WRITE; + static const S32 READ_WRITE; + static const S32 APPEND; + +protected: + LLAssetType::EType mFileType; + BOOL mReadComplete; + LLUUID mFileID; + S32 mPosition; + S32 mMode; + S32 mBytesRead; +}; + +#endif // LL_LLDISKCACHE_H -- cgit v1.3 From 96e2873bfa2c6b8823aed3b4190c43cd5dab54e6 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Thu, 24 Sep 2020 10:23:39 -0700 Subject: Rename lldiskcache.* to llfilesystem.* - i think this is the right name since it's responsible for performing file system operations and (will eventually) delegrate to a separate disk cache component to save/load data and keep track of metadata etc. --- indra/llfilesystem/lldiskcache.cpp | 387 ------------------------------------ indra/llfilesystem/lldiskcache.h | 82 -------- indra/llfilesystem/llfilesystem.cpp | 387 ++++++++++++++++++++++++++++++++++++ indra/llfilesystem/llfilesystem.h | 82 ++++++++ 4 files changed, 469 insertions(+), 469 deletions(-) delete mode 100644 indra/llfilesystem/lldiskcache.cpp delete mode 100644 indra/llfilesystem/lldiskcache.h create mode 100644 indra/llfilesystem/llfilesystem.cpp create mode 100644 indra/llfilesystem/llfilesystem.h (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp deleted file mode 100644 index af93049e07..0000000000 --- a/indra/llfilesystem/lldiskcache.cpp +++ /dev/null @@ -1,387 +0,0 @@ -/** - * @file lldiskcache.cpp - * @brief Implementation of virtual file - * - * $LicenseInfo:firstyear=2002&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#include "linden_common.h" - -#include "lldiskcache.h" - -#include "llerror.h" -#include "llthread.h" -#include "lltimer.h" -#include "llfasttimer.h" -#include "llmemory.h" - -#include -#include "lldir.h" - -const S32 LLDiskCache::READ = 0x00000001; -const S32 LLDiskCache::WRITE = 0x00000002; -const S32 LLDiskCache::READ_WRITE = 0x00000003; // LLDiskCache::READ & LLDiskCache::WRITE -const S32 LLDiskCache::APPEND = 0x00000006; // 0x00000004 & LLDiskCache::WRITE - -static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); - -LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) -{ - mFileType = file_type; - mFileID = file_id; - mPosition = 0; - mBytesRead = 0; - mReadComplete = FALSE; - mMode = mode; -} - -LLDiskCache::~LLDiskCache() -{ -} - -const std::string assetTypeToString(LLAssetType::EType at) -{ - /** - * Make use of the C++17 (or is it 14) feature that allows - * for inline initialization of an std::map<> - */ - typedef std::map asset_type_to_name_t; - asset_type_to_name_t asset_type_to_name = - { - { LLAssetType::AT_TEXTURE, "TEXTURE" }, - { LLAssetType::AT_SOUND, "SOUND" }, - { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, - { LLAssetType::AT_LANDMARK, "LANDMARK" }, - { LLAssetType::AT_SCRIPT, "SCRIPT" }, - { LLAssetType::AT_CLOTHING, "CLOTHING" }, - { LLAssetType::AT_OBJECT, "OBJECT" }, - { LLAssetType::AT_NOTECARD, "NOTECARD" }, - { LLAssetType::AT_CATEGORY, "CATEGORY" }, - { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, - { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, - { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, - { LLAssetType::AT_BODYPART, "BODYPART" }, - { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, - { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, - { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, - { LLAssetType::AT_ANIMATION, "ANIMATION" }, - { LLAssetType::AT_GESTURE, "GESTURE" }, - { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, - { LLAssetType::AT_LINK, "LINK" }, - { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, - { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, - { LLAssetType::AT_WIDGET, "WIDGET" }, - { LLAssetType::AT_PERSON, "PERSON" }, - { LLAssetType::AT_MESH, "MESH" }, - { LLAssetType::AT_SETTINGS, "SETTINGS" }, - { LLAssetType::AT_UNKNOWN, "UNKNOWN" } - }; - - asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); - if (iter != asset_type_to_name.end()) - { - return iter->second; - } - - return std::string("UNKNOWN"); -} - -const std::string idToFilepath(const std::string id, LLAssetType::EType at) -{ - /** - * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of - * course, will be greatly expanded upon - */ - std::ostringstream ss; - ss << "00cache_"; - ss << id; - ss << "_"; - ss << assetTypeToString(at); - ss << ".txt"; - - const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ss.str()); - - return filepath; -} - -// static -bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) -{ - std::string id_str; - file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - - std::ifstream file(filename, std::ios::binary); - if (file.is_open()) - { - file.seekg(0, std::ios::end); - return file.tellg() > 0; - } - return false; -} - -// static -bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) -{ - std::string id_str; - file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - - std::remove(filename.c_str()); - - return true; -} - -// static -bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, - const LLUUID &new_file_id, const LLAssetType::EType new_file_type) -{ - std::string old_id_str; - old_file_id.toString(old_id_str); - const std::string old_filename = idToFilepath(old_id_str, old_file_type); - - std::string new_id_str; - new_file_id.toString(new_id_str); - const std::string new_filename = idToFilepath(new_id_str, new_file_type); - - if (std::rename(old_filename.c_str(), new_filename.c_str())) - { - // We would like to return FALSE here indicating the operation - // failed but the original code does not and doing so seems to - // break a lot of things so we go with the flow... - //return FALSE; - } - - return TRUE; -} - -// static -S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) -{ - std::string id_str; - file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - - S32 file_size = 0; - std::ifstream file(filename, std::ios::binary); - if (file.is_open()) - { - file.seekg(0, std::ios::end); - file_size = file.tellg(); - } - - return file_size; -} - -BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) -{ - BOOL success = TRUE; - - mReadComplete = FALSE; - - std::string id; - mFileID.toString(id); - const std::string filename = idToFilepath(id, mFileType); - - std::ifstream file(filename, std::ios::binary); - if (file.is_open()) - { - file.seekg(mPosition, std::ios::beg); - - file.read((char*)buffer, bytes); - - if (file) - { - mBytesRead = bytes; - } - else - { - mBytesRead = file.gcount(); - } - - file.close(); - - mPosition += mBytesRead; - if (!mBytesRead) - { - success = FALSE; - } - - mReadComplete = TRUE; - } - - return success; -} - -BOOL LLDiskCache::isReadComplete() -{ - if (mReadComplete) - { - return TRUE; - } - - return FALSE; -} - -S32 LLDiskCache::getLastBytesRead() -{ - return mBytesRead; -} - -BOOL LLDiskCache::eof() -{ - return mPosition >= getSize(); -} - -BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) -{ - std::string id_str; - mFileID.toString(id_str); - const std::string filename = idToFilepath(id_str, mFileType); - - BOOL success = FALSE; - - if (mMode == APPEND) - { - std::ofstream ofs(filename, std::ios::app | std::ios::binary); - if (ofs) - { - ofs.write((const char*)buffer, bytes); - - success = TRUE; - } - } - else - { - std::ofstream ofs(filename, std::ios::binary); - if (ofs) - { - ofs.write((const char*)buffer, bytes); - - mPosition += bytes; - - success = TRUE; - } - } - - return success; -} - -//static -BOOL LLDiskCache::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) -{ - LLDiskCache file(uuid, type, LLDiskCache::WRITE); - file.setMaxSize(bytes); - return file.write(buffer, bytes); -} - -BOOL LLDiskCache::seek(S32 offset, S32 origin) -{ - if (-1 == origin) - { - origin = mPosition; - } - - S32 new_pos = origin + offset; - - S32 size = getSize(); - - if (new_pos > size) - { - LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; - - mPosition = size; - return FALSE; - } - else if (new_pos < 0) - { - LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; - - mPosition = 0; - return FALSE; - } - - mPosition = new_pos; - return TRUE; -} - -S32 LLDiskCache::tell() const -{ - return mPosition; -} - -S32 LLDiskCache::getSize() -{ - return LLDiskCache::getFileSize(mFileID, mFileType); -} - -S32 LLDiskCache::getMaxSize() -{ - // offer up a huge size since we don't care what the max is - return INT_MAX; -} - -BOOL LLDiskCache::setMaxSize(S32 size) -{ - // we don't care what the max size is so we do nothing - // and return true to indicate all was okay - return TRUE; -} - -BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type) -{ - LLDiskCache::renameFile(mFileID, mFileType, new_id, new_type); - - mFileID = new_id; - mFileType = new_type; - - return TRUE; -} - -BOOL LLDiskCache::remove() -{ - LLDiskCache::removeFile(mFileID, mFileType); - - return TRUE; -} - -// static -void LLDiskCache::initClass() -{ -} - -// static -void LLDiskCache::cleanupClass() -{ -} - -bool LLDiskCache::isLocked() -{ - // I don't think we care about this test since there is no locking - // TODO: remove this function and calling sites? - return FALSE; -} - -void LLDiskCache::waitForLock() -{ - // TODO: remove this function and calling sites? -} diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h deleted file mode 100644 index 7ad06a8689..0000000000 --- a/indra/llfilesystem/lldiskcache.h +++ /dev/null @@ -1,82 +0,0 @@ -/** - * @file lldiskcacke.h - * @brief Definition of virtual file - * - * $LicenseInfo:firstyear=2002&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#ifndef LL_LLDISKCACHE_H -#define LL_LLDISKCACHE_H - -#include "lluuid.h" -#include "llassettype.h" - -class LLDiskCache -{ -public: - LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLDiskCache::READ); - ~LLDiskCache(); - - BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ - BOOL isReadComplete(); - S32 getLastBytesRead(); - BOOL eof(); - - BOOL write(const U8 *buffer, S32 bytes); - static BOOL writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type); - BOOL seek(S32 offset, S32 origin = -1); - S32 tell() const; - - S32 getSize(); - S32 getMaxSize(); - BOOL setMaxSize(S32 size); - BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); - BOOL remove(); - - bool isLocked(); - void waitForLock(); - - static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); - static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); - static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, - const LLUUID &new_file_id, const LLAssetType::EType new_file_type); - static S32 getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type); - - static void initClass(); - static void cleanupClass(); - -public: - static const S32 READ; - static const S32 WRITE; - static const S32 READ_WRITE; - static const S32 APPEND; - -protected: - LLAssetType::EType mFileType; - BOOL mReadComplete; - LLUUID mFileID; - S32 mPosition; - S32 mMode; - S32 mBytesRead; -}; - -#endif // LL_LLDISKCACHE_H diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp new file mode 100644 index 0000000000..af93049e07 --- /dev/null +++ b/indra/llfilesystem/llfilesystem.cpp @@ -0,0 +1,387 @@ +/** + * @file lldiskcache.cpp + * @brief Implementation of virtual file + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include "lldiskcache.h" + +#include "llerror.h" +#include "llthread.h" +#include "lltimer.h" +#include "llfasttimer.h" +#include "llmemory.h" + +#include +#include "lldir.h" + +const S32 LLDiskCache::READ = 0x00000001; +const S32 LLDiskCache::WRITE = 0x00000002; +const S32 LLDiskCache::READ_WRITE = 0x00000003; // LLDiskCache::READ & LLDiskCache::WRITE +const S32 LLDiskCache::APPEND = 0x00000006; // 0x00000004 & LLDiskCache::WRITE + +static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); + +LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) +{ + mFileType = file_type; + mFileID = file_id; + mPosition = 0; + mBytesRead = 0; + mReadComplete = FALSE; + mMode = mode; +} + +LLDiskCache::~LLDiskCache() +{ +} + +const std::string assetTypeToString(LLAssetType::EType at) +{ + /** + * Make use of the C++17 (or is it 14) feature that allows + * for inline initialization of an std::map<> + */ + typedef std::map asset_type_to_name_t; + asset_type_to_name_t asset_type_to_name = + { + { LLAssetType::AT_TEXTURE, "TEXTURE" }, + { LLAssetType::AT_SOUND, "SOUND" }, + { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, + { LLAssetType::AT_LANDMARK, "LANDMARK" }, + { LLAssetType::AT_SCRIPT, "SCRIPT" }, + { LLAssetType::AT_CLOTHING, "CLOTHING" }, + { LLAssetType::AT_OBJECT, "OBJECT" }, + { LLAssetType::AT_NOTECARD, "NOTECARD" }, + { LLAssetType::AT_CATEGORY, "CATEGORY" }, + { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, + { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, + { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, + { LLAssetType::AT_BODYPART, "BODYPART" }, + { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, + { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, + { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, + { LLAssetType::AT_ANIMATION, "ANIMATION" }, + { LLAssetType::AT_GESTURE, "GESTURE" }, + { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, + { LLAssetType::AT_LINK, "LINK" }, + { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, + { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, + { LLAssetType::AT_WIDGET, "WIDGET" }, + { LLAssetType::AT_PERSON, "PERSON" }, + { LLAssetType::AT_MESH, "MESH" }, + { LLAssetType::AT_SETTINGS, "SETTINGS" }, + { LLAssetType::AT_UNKNOWN, "UNKNOWN" } + }; + + asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); + if (iter != asset_type_to_name.end()) + { + return iter->second; + } + + return std::string("UNKNOWN"); +} + +const std::string idToFilepath(const std::string id, LLAssetType::EType at) +{ + /** + * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of + * course, will be greatly expanded upon + */ + std::ostringstream ss; + ss << "00cache_"; + ss << id; + ss << "_"; + ss << assetTypeToString(at); + ss << ".txt"; + + const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ss.str()); + + return filepath; +} + +// static +bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string filename = idToFilepath(id_str, file_type); + + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(0, std::ios::end); + return file.tellg() > 0; + } + return false; +} + +// static +bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string filename = idToFilepath(id_str, file_type); + + std::remove(filename.c_str()); + + return true; +} + +// static +bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, + const LLUUID &new_file_id, const LLAssetType::EType new_file_type) +{ + std::string old_id_str; + old_file_id.toString(old_id_str); + const std::string old_filename = idToFilepath(old_id_str, old_file_type); + + std::string new_id_str; + new_file_id.toString(new_id_str); + const std::string new_filename = idToFilepath(new_id_str, new_file_type); + + if (std::rename(old_filename.c_str(), new_filename.c_str())) + { + // We would like to return FALSE here indicating the operation + // failed but the original code does not and doing so seems to + // break a lot of things so we go with the flow... + //return FALSE; + } + + return TRUE; +} + +// static +S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) +{ + std::string id_str; + file_id.toString(id_str); + const std::string filename = idToFilepath(id_str, file_type); + + S32 file_size = 0; + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(0, std::ios::end); + file_size = file.tellg(); + } + + return file_size; +} + +BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) +{ + BOOL success = TRUE; + + mReadComplete = FALSE; + + std::string id; + mFileID.toString(id); + const std::string filename = idToFilepath(id, mFileType); + + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) + { + file.seekg(mPosition, std::ios::beg); + + file.read((char*)buffer, bytes); + + if (file) + { + mBytesRead = bytes; + } + else + { + mBytesRead = file.gcount(); + } + + file.close(); + + mPosition += mBytesRead; + if (!mBytesRead) + { + success = FALSE; + } + + mReadComplete = TRUE; + } + + return success; +} + +BOOL LLDiskCache::isReadComplete() +{ + if (mReadComplete) + { + return TRUE; + } + + return FALSE; +} + +S32 LLDiskCache::getLastBytesRead() +{ + return mBytesRead; +} + +BOOL LLDiskCache::eof() +{ + return mPosition >= getSize(); +} + +BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) +{ + std::string id_str; + mFileID.toString(id_str); + const std::string filename = idToFilepath(id_str, mFileType); + + BOOL success = FALSE; + + if (mMode == APPEND) + { + std::ofstream ofs(filename, std::ios::app | std::ios::binary); + if (ofs) + { + ofs.write((const char*)buffer, bytes); + + success = TRUE; + } + } + else + { + std::ofstream ofs(filename, std::ios::binary); + if (ofs) + { + ofs.write((const char*)buffer, bytes); + + mPosition += bytes; + + success = TRUE; + } + } + + return success; +} + +//static +BOOL LLDiskCache::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) +{ + LLDiskCache file(uuid, type, LLDiskCache::WRITE); + file.setMaxSize(bytes); + return file.write(buffer, bytes); +} + +BOOL LLDiskCache::seek(S32 offset, S32 origin) +{ + if (-1 == origin) + { + origin = mPosition; + } + + S32 new_pos = origin + offset; + + S32 size = getSize(); + + if (new_pos > size) + { + LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; + + mPosition = size; + return FALSE; + } + else if (new_pos < 0) + { + LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; + + mPosition = 0; + return FALSE; + } + + mPosition = new_pos; + return TRUE; +} + +S32 LLDiskCache::tell() const +{ + return mPosition; +} + +S32 LLDiskCache::getSize() +{ + return LLDiskCache::getFileSize(mFileID, mFileType); +} + +S32 LLDiskCache::getMaxSize() +{ + // offer up a huge size since we don't care what the max is + return INT_MAX; +} + +BOOL LLDiskCache::setMaxSize(S32 size) +{ + // we don't care what the max size is so we do nothing + // and return true to indicate all was okay + return TRUE; +} + +BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type) +{ + LLDiskCache::renameFile(mFileID, mFileType, new_id, new_type); + + mFileID = new_id; + mFileType = new_type; + + return TRUE; +} + +BOOL LLDiskCache::remove() +{ + LLDiskCache::removeFile(mFileID, mFileType); + + return TRUE; +} + +// static +void LLDiskCache::initClass() +{ +} + +// static +void LLDiskCache::cleanupClass() +{ +} + +bool LLDiskCache::isLocked() +{ + // I don't think we care about this test since there is no locking + // TODO: remove this function and calling sites? + return FALSE; +} + +void LLDiskCache::waitForLock() +{ + // TODO: remove this function and calling sites? +} diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h new file mode 100644 index 0000000000..7ad06a8689 --- /dev/null +++ b/indra/llfilesystem/llfilesystem.h @@ -0,0 +1,82 @@ +/** + * @file lldiskcacke.h + * @brief Definition of virtual file + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLDISKCACHE_H +#define LL_LLDISKCACHE_H + +#include "lluuid.h" +#include "llassettype.h" + +class LLDiskCache +{ +public: + LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLDiskCache::READ); + ~LLDiskCache(); + + BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ + BOOL isReadComplete(); + S32 getLastBytesRead(); + BOOL eof(); + + BOOL write(const U8 *buffer, S32 bytes); + static BOOL writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type); + BOOL seek(S32 offset, S32 origin = -1); + S32 tell() const; + + S32 getSize(); + S32 getMaxSize(); + BOOL setMaxSize(S32 size); + BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); + BOOL remove(); + + bool isLocked(); + void waitForLock(); + + static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); + static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); + static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, + const LLUUID &new_file_id, const LLAssetType::EType new_file_type); + static S32 getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type); + + static void initClass(); + static void cleanupClass(); + +public: + static const S32 READ; + static const S32 WRITE; + static const S32 READ_WRITE; + static const S32 APPEND; + +protected: + LLAssetType::EType mFileType; + BOOL mReadComplete; + LLUUID mFileID; + S32 mPosition; + S32 mMode; + S32 mBytesRead; +}; + +#endif // LL_LLDISKCACHE_H -- cgit v1.3 From 6be1f88a5ef99e1e40bb5701a250ba0728f56005 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Thu, 24 Sep 2020 14:45:39 -0700 Subject: Complete the change from lldiskcache -> llfilesystem and then addition of new lldiskcache implementation --- indra/llappearance/lltexlayer.cpp | 1 - indra/llaudio/llaudiodecodemgr.cpp | 14 +- indra/llaudio/llaudioengine.cpp | 4 +- indra/llcharacter/llkeyframemotion.cpp | 6 +- indra/llfilesystem/CMakeLists.txt | 3 +- indra/llfilesystem/lldiskcache.cpp | 773 ++++++++++++++++++++++++++++++ indra/llfilesystem/lldiskcache.h | 130 +++++ indra/llfilesystem/llfilesystem.cpp | 79 ++- indra/llfilesystem/llfilesystem.h | 19 +- indra/llmessage/llassetstorage.cpp | 28 +- indra/llmessage/llcorehttputil.cpp | 4 +- indra/llmessage/lltransfersourceasset.cpp | 6 +- indra/llmessage/lltransfersourceasset.h | 2 +- indra/llmessage/lltransfertargetvfile.cpp | 8 +- indra/llmessage/lltransfertargetvfile.h | 4 +- indra/llmessage/llxfer_vfile.cpp | 22 +- indra/llmessage/llxfer_vfile.h | 4 +- indra/llui/llviewereventrecorder.h | 1 - indra/newview/llappviewer.cpp | 6 +- indra/newview/llcompilequeue.cpp | 2 +- indra/newview/llfloaterauction.cpp | 6 +- indra/newview/llfloaterbvhpreview.cpp | 4 +- indra/newview/llfloatermodelpreview.cpp | 2 +- indra/newview/llfloaterregioninfo.cpp | 4 +- indra/newview/llfloaterreporter.cpp | 4 +- indra/newview/llfloatertos.cpp | 2 +- indra/newview/llgesturemgr.cpp | 4 +- indra/newview/lllandmarklist.cpp | 4 +- indra/newview/llmeshrepository.cpp | 22 +- indra/newview/lloutfitgallery.cpp | 2 +- indra/newview/llpostcard.cpp | 2 +- indra/newview/llpreviewgesture.cpp | 6 +- indra/newview/llpreviewnotecard.cpp | 10 +- indra/newview/llpreviewscript.cpp | 6 +- indra/newview/llsettingsvo.cpp | 4 +- indra/newview/llsnapshotlivepreview.cpp | 4 +- indra/newview/llviewerassetstorage.cpp | 12 +- indra/newview/llviewerassetstorage.h | 2 +- indra/newview/llviewerassetupload.cpp | 8 +- indra/newview/llviewermenufile.cpp | 1 - indra/newview/llviewermessage.cpp | 4 +- indra/newview/llviewerstats.cpp | 1 - indra/newview/llviewertexlayer.cpp | 1 - indra/newview/llviewertexture.cpp | 1 - indra/newview/llviewertexture.h | 2 +- indra/newview/llviewertexturelist.cpp | 2 +- 46 files changed, 1068 insertions(+), 168 deletions(-) create mode 100644 indra/llfilesystem/lldiskcache.cpp create mode 100644 indra/llfilesystem/lldiskcache.h (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llappearance/lltexlayer.cpp b/indra/llappearance/lltexlayer.cpp index 7360c1acd7..1348fb0763 100644 --- a/indra/llappearance/lltexlayer.cpp +++ b/indra/llappearance/lltexlayer.cpp @@ -33,7 +33,6 @@ #include "llimagej2c.h" #include "llimagetga.h" #include "lldir.h" -#include "lldiskcache.h" #include "lltexlayerparams.h" #include "lltexturemanagerbridge.h" #include "lllocaltextureobject.h" diff --git a/indra/llaudio/llaudiodecodemgr.cpp b/indra/llaudio/llaudiodecodemgr.cpp index fcffea42a4..ff0aa6e76e 100644 --- a/indra/llaudio/llaudiodecodemgr.cpp +++ b/indra/llaudio/llaudiodecodemgr.cpp @@ -29,7 +29,7 @@ #include "llaudioengine.h" #include "lllfsthread.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llstring.h" #include "lldir.h" #include "llendianswizzle.h" @@ -93,14 +93,14 @@ protected: std::string mOutFilename; LLLFSThread::handle_t mFileHandle; - LLDiskCache *mInFilep; + LLFileSystem *mInFilep; OggVorbis_File mVF; S32 mCurrentSection; }; size_t cache_read(void *ptr, size_t size, size_t nmemb, void *datasource) { - LLDiskCache *file = (LLDiskCache *)datasource; + LLFileSystem *file = (LLFileSystem *)datasource; if (file->read((U8*)ptr, (S32)(size * nmemb))) /*Flawfinder: ignore*/ { @@ -115,7 +115,7 @@ size_t cache_read(void *ptr, size_t size, size_t nmemb, void *datasource) S32 cache_seek(void *datasource, ogg_int64_t offset, S32 whence) { - LLDiskCache *file = (LLDiskCache *)datasource; + LLFileSystem *file = (LLFileSystem *)datasource; // cache has 31-bit files if (offset > S32_MAX) @@ -151,14 +151,14 @@ S32 cache_seek(void *datasource, ogg_int64_t offset, S32 whence) S32 cache_close (void *datasource) { - LLDiskCache *file = (LLDiskCache *)datasource; + LLFileSystem *file = (LLFileSystem *)datasource; delete file; return 0; } long cache_tell (void *datasource) { - LLDiskCache *file = (LLDiskCache *)datasource; + LLFileSystem *file = (LLFileSystem *)datasource; return file->tell(); } @@ -198,7 +198,7 @@ BOOL LLVorbisDecodeState::initDecode() LL_DEBUGS("AudioEngine") << "Initing decode from vfile: " << mUUID << LL_ENDL; - mInFilep = new LLDiskCache(mUUID, LLAssetType::AT_SOUND); + mInFilep = new LLFileSystem(mUUID, LLAssetType::AT_SOUND); if (!mInFilep || !mInFilep->getSize()) { LL_WARNS("AudioEngine") << "unable to open vorbis source vfile for reading" << LL_ENDL; diff --git a/indra/llaudio/llaudioengine.cpp b/indra/llaudio/llaudioengine.cpp index 9dd752f492..d35f249973 100644 --- a/indra/llaudio/llaudioengine.cpp +++ b/indra/llaudio/llaudioengine.cpp @@ -35,7 +35,7 @@ #include "sound_ids.h" // temporary hack for min/max distances -#include "lldiskcache.h" +#include "llfilesystem.h" #include "lldir.h" #include "llaudiodecodemgr.h" #include "llassetstorage.h" @@ -1015,7 +1015,7 @@ bool LLAudioEngine::hasDecodedFile(const LLUUID &uuid) bool LLAudioEngine::hasLocalFile(const LLUUID &uuid) { // See if it's in the cache. - bool have_local = LLDiskCache::getExists(uuid, LLAssetType::AT_SOUND); + bool have_local = LLFileSystem::getExists(uuid, LLAssetType::AT_SOUND); LL_DEBUGS("AudioEngine") << "sound uuid " << uuid << " exists in cache" << LL_ENDL; return have_local; } diff --git a/indra/llcharacter/llkeyframemotion.cpp b/indra/llcharacter/llkeyframemotion.cpp index d1ac336fc1..fe9de30f0a 100644 --- a/indra/llcharacter/llkeyframemotion.cpp +++ b/indra/llcharacter/llkeyframemotion.cpp @@ -39,9 +39,9 @@ #include "llendianswizzle.h" #include "llkeyframemotion.h" #include "llquantize.h" -#include "lldiskcache.h" #include "m3math.h" #include "message.h" +#include "llfilesystem.h" //----------------------------------------------------------------------------- // Static Definitions @@ -559,7 +559,7 @@ LLMotion::LLMotionInitStatus LLKeyframeMotion::onInitialize(LLCharacter *charact S32 anim_file_size; BOOL success = FALSE; - LLDiskCache* anim_file = new LLDiskCache(mID, LLAssetType::AT_ANIMATION); + LLFileSystem* anim_file = new LLFileSystem(mID, LLAssetType::AT_ANIMATION); if (!anim_file || !anim_file->getSize()) { delete anim_file; @@ -2324,7 +2324,7 @@ void LLKeyframeMotion::onLoadComplete(const LLUUID& asset_uuid, // asset already loaded return; } - LLDiskCache file(asset_uuid, type, LLDiskCache::READ); + LLFileSystem file(asset_uuid, type, LLFileSystem::READ); S32 size = file.getSize(); U8* buffer = new U8[size]; diff --git a/indra/llfilesystem/CMakeLists.txt b/indra/llfilesystem/CMakeLists.txt index 306b483097..d1dece5bba 100644 --- a/indra/llfilesystem/CMakeLists.txt +++ b/indra/llfilesystem/CMakeLists.txt @@ -18,16 +18,17 @@ set(llfilesystem_SOURCE_FILES lldiriterator.cpp lllfsthread.cpp lldiskcache.cpp + llfilesystem.cpp ) set(llfilesystem_HEADER_FILES CMakeLists.txt - lldir.h lldirguard.h lldiriterator.h lllfsthread.h lldiskcache.h + llfilesystem.h ) if (DARWIN) diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp new file mode 100644 index 0000000000..72982db069 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.cpp @@ -0,0 +1,773 @@ +/** + * @file lldiskcache.cpp + * @brief The SQLite based disk cache implementation. + * @Note Eventually, this component might split into an interface + * file and multiple implemtations but for now, this is the + * only one so I think it's okay to combine everything. + * + * $LicenseInfo:firstyear=2009&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2020, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#if (defined(LL_WINDOWS) || defined(LL_LINUX) || defined(LL_DARWIN)) +#include "linden_common.h" +#endif + +#include "lldiskcache.h" + +#include +#include +#include +#include +#include + +/////////////////////////////////////////////////////////////////////////////// +// +llDiskCache::llDiskCache() : + mDataStorePath("") +{ +} + +llDiskCache::~llDiskCache() +{ +} + +/////////////////////////////////////////////////////////////////////////////// +// Opens the database - typically done when the application starts +// and is complementary to close() which is called when the application +// is finisahed and exits. +// Pass in the folder and filename of the SQLite database you want to +// use or create (file doesn't have to exist but the folder must) +// Returns true or false and writes a message to console on error +bool llDiskCache::open(const std::string db_folder, const std::string db_filename) +{ + mDataStorePath = db_folder; + std::string db_pathname = makeFullPath(db_filename); + + // simple flags for the moment - these will likely be extended + // later on to support the SQLite mutex model for reading/writing + // simultaneously - perhaps when we look at supporting textures too + int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + + if (sqlite3_open_v2( + db_pathname.c_str(), + &mDb, + flags, + nullptr // Name of VFS module to use + ) != SQLITE_OK) + { + printError(__FUNCDNAME__ , "open_v2", true); + close(); + return false; + } + + // I elected to store the code that generates the statement + // in sepsrate functions throughout - this seemed like a cleaner + // approach than having hundreds of stmt << "REPLACE AS" lines + // interspersed in the logic code. They are all prefixed with + // 'sqlCompose' and followed by a short description. + const std::string stmt = sqlComposeCreateTable(); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + // Not sure if we need close here - if open fails, then we + // perhaps don't need to close it - TODO: look in SQLite docs + close(); + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Determines if an entry exists. +// Pass in the id and a variable that will indicate existance +// Returns true/false if function succeeded and the boolean +// you pass in will be set to true/false if entry exists or not +bool llDiskCache::exists(const std::string id, bool& exists) +{ + if (!mDb) + { + printError(__FUNCDNAME__ , "mDb is invalid", false); + return false; + } + + if (id.empty()) + { + printError(__FUNCDNAME__ , "id is empty", false); + return false; + } + + // As well as the separate function to compose the SQL statement, + // worth pointing out that the code to 'prepare' and 'step' the + // SQLite "prepared statement" has been factored out into its own + // function and used in several other functions. + const std::string stmt = sqlComposeExists(id); + sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_ROW); + if (! prepared_statement) + { + return false; + } + + int result_column_index = 0; + int result_count = sqlite3_column_int(prepared_statement, result_column_index); + if (sqlite3_finalize(prepared_statement) != SQLITE_OK) + { + printError(__FUNCDNAME__ , "sqlite3_finalize()", true); + return false; + } + + // given the uniqueness of the ID, this can only ever be + // either 1 or 0 so this might be a bit confusing + exists = result_count > 0 ? true : false; + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id (likely a UUID + decoration but can be any string), a +// pointer to a blob of binary data along with its length, write the +// entry to the cache +// Returns true/false for success/failure +bool llDiskCache::put(const std::string id, + char* binary_data, + int binary_data_size) +{ + if (!mDb) + { + printError(__FUNCDNAME__ , "mDb is invalid", false); + return false; + } + + if (id.empty()) + { + printError(__FUNCDNAME__ , "id is empty", false); + return false; + } + + // < 0 is obvious but we assert the data must be at least 1 byte long + if (binary_data_size <= 0) + { + printError(__FUNCDNAME__ , "size of binary file to write is invalid", false); + return false; + } + + // we generate a unique filename for the actual data itself + // which is stored on disk directly and not in the database. + // TODO: consider making the filename more like the ID passed in + // although the problem with that is we would have to parse the id + // to remove invalid filename chars, consider length etc. As it + // stands, we can write a simple SQL statement to return the filename + // given the ID. + const std::string filename = makeUniqueFilename(); + const std::string filepath = makeFullPath(filename); + std::ofstream file(filepath, std::ios::out | std::ios::binary); + if (! file.is_open()) + { + std::ostringstream error; + error << "Unable to open " << filepath << " for writing"; + printError(__FUNCDNAME__ , error.str(), false); + return false; + } + + file.write((char*)binary_data, binary_data_size); + file.close(); + + // I think is a catchall "wasn't able to write the file to disk" + // conditional but should revisit when we hook this up to viewer + // code to make sure - we never want to write bad/no data to the + // disk and not indicate something has gone wrong + if (file.bad()) + { + std::ostringstream error; + error << "Unable to write " << binary_data_size; + error << " bytes to " << filepath; + printError(__FUNCDNAME__ , error.str(), false); + + return false; + } + + // this is where the filename/size is written to the database along + // with the current date/time for the created/last access times + const std::string stmt = sqlComposePut(id, filename, binary_data_size); + if (! sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_DONE)) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id (likely a UUID + decoration but can be any string), the +// address of a pointer that will be used to allocate memory and a +// varialble that will contain the length of the data, get an item from +// the cache. Note that the memory buffer returned belongs to the calling +// function and it is its responsiblity to clean it up when it's no +// longer needed. +// Returns true/false for success/failure +const bool llDiskCache::get(const std::string id, + char** mem_buffer, + int& mem_buffer_size) +{ + // Check if the entry exists first to avoid dealing with a bunch + // of conditions that look like failure but aren't in the main code. + // Note exists() is a public method and also tests for mDB and id + // being valid so we can safely put this about the same tests + // in this function + bool get_exists = false; + if (! exists(id, get_exists)) + { + return false; + } + if (! get_exists) + { + return false; + } + + if (!mDb) + { + printError(__FUNCDNAME__ , "mDb is invalid", false); + return false; + } + + if (id.empty()) + { + printError(__FUNCDNAME__ , "id is empty", false); + return false; + } + + const std::string stmt_select = sqlComposeGetSelect(id); + sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt_select, SQLITE_ROW); + if (! prepared_statement) + { + return false; + } + + int result_column_index = 0; + const unsigned char* text = sqlite3_column_text(prepared_statement, result_column_index); + if (text == nullptr) + { + printError(__FUNCDNAME__ , "filename is nullptr", true); + return false; + } + const std::string filename = std::string(reinterpret_cast(text)); + const std::string filepath = makeFullPath(filename); + + result_column_index = 1; + int filesize_db = sqlite3_column_int(prepared_statement, result_column_index); + if (filesize_db <= 0) + { + printError(__FUNCDNAME__ , "filesize is invalid", true); + return false; + } + + if (sqlite3_finalize(prepared_statement) != SQLITE_OK) + { + printError(__FUNCDNAME__ , "sqlite3_finalize()", true); + return false; + } + + // Now we have the fiename, we can read the file from disk + std::ifstream file(filepath, std::ios::in | std::ios::binary | std::ios::ate); + if (! file.is_open()) + { + std::ostringstream error; + error << "Unable to open " << filepath << " for reading"; + printError(__FUNCDNAME__ , error.str(), false); + return false; + } + + // we get the expected filesize from the database but we can also + // get it (easily) when we read the file from the disk. We compare + // the two and return false if they don't match + std::streampos filesize_file = file.tellg(); + if (filesize_db != filesize_file) + { + std::ostringstream error; + error << "File size from DB (" << filesize_db << ")"; + error << " and "; + error << "file size from file (" << filesize_file << ")"; + error << " in file " << filepath << " are different"; + printError(__FUNCDNAME__ , error.str(), false); + + return false; + } + + // doest matter if we choose DB or file version - they must be + // identical if we get this far - just used for clarity + int filesize = filesize_db; + + // allocate a block of memory that we pass back for the calling + // function to use then delete when it's no longer needed + *mem_buffer = new char[filesize]; + mem_buffer_size = filesize; + + file.seekg(0, std::ios::beg); + file.read(*mem_buffer, filesize); + file.close(); + + if (file.bad()) + { + std::ostringstream error; + error << "Unable to read " << filesize; + error << " bytes from " << filepath; + printError(__FUNCDNAME__ , error.str(), false); + + return false; + } + + // here we update the count of times the file is accessed so + // we can keep track of how many times it's been requested. + // This will be useful for metrics and perhaps determining + // if a file should not be purged even though its age + // might suggest that it should. + // In addition, this is where the time of last access is updated + // in the database and that us used to determine what is purged + // in an LRU fashion when the purge function is called. + const std::string stmt_update = sqlComposeGetUpdate(id); + if (! sqliteExec(stmt_update, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Purges the database of older entries using an LRU approach. +// Pass in the number of entries to retain. +// This is called now after open to "clean up" the cache when the +// application starts. +// TODO: IMPORTANT: compose a list of files that will be deleted +// and delete them from disk too - not just from the DB +bool llDiskCache::purge(int num_entries) +{ + if (num_entries < 0) + { + printError(__FUNCDNAME__ , "number of entries to purge is invalid", false); + return false; + } + + // find the rows affected and get the filenames for them +//swww + + // delete oldest entries leaving the correct number in place + const std::string stmt = sqlComposePurge(num_entries); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Call at application shutdown +void llDiskCache::close() +{ + sqlite3_close(mDb); +} + +/////////////////////////////////////////////////////////////////////////////// +// Determine the version of SQLite in use +// TODO: make this a static so we can get to it from the Viewer About +// box without instantiating the whole thing. +const std::string llDiskCache::dbVersion() +{ + std::ostringstream version; + + version << sqlite3_libversion(); + + return version.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id, return the matching filename +const std::string llDiskCache::getFilenameById(const std::string id) +{ + // TODO: + return std::string(); +} + +/////////////////////////////////////////////////////////////////////////////// +// Given an id, return the number of times that entry has been +// accessed from the cache +const int llDiskCache::getAccessCountById(const std::string id) +{ + // TODO: + return -1; +} + +/////////////////////////////////////////////////////////////////////////////// +// Return the number of entries currently in the cache as well as +// the maximum possible entries. +void llDiskCache::getNumEntries(int& num_entries, int& max_entries) +{ + num_entries = -1; + max_entries = -1; +} + +/////////////////////////////////////////////////////////////////////////////// +// Wraps the sqlite3_exec(..) used in many places +bool llDiskCache::sqliteExec(const std::string stmt, + const std::string funcname) +{ + if (sqlite3_exec( + mDb, + stmt.c_str(), + nullptr, // Callback function (unused) + nullptr, // 1st argument to callback (unused) + nullptr // Error msg written here (unused) + ) != SQLITE_OK) + { + printError(funcname, "sqlite3_exec", true); + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Wraps the sqlite3_prepare_v2 and sqlite3_step calls used in many places +sqlite3_stmt* llDiskCache::sqlitePrepareStep(const std::string funcname, + const std::string stmt, + int sqlite_success_condition) +{ + sqlite3_stmt* prepared_statement; + + if (sqlite3_prepare_v2( + mDb, + stmt.c_str(), + -1, // Maximum length of zSql in bytes. + &prepared_statement, + 0 // OUT: Pointer to unused portion of zSql + ) != SQLITE_OK) + { + printError(funcname, "sqlite3_prepare_v2", true); + return nullptr; + } + + if (sqlite3_step(prepared_statement) != sqlite_success_condition) + { + printError(funcname, "sqlite3_step", true); + sqlite3_finalize(prepared_statement); + return nullptr; + } + return prepared_statement; +} + +/////////////////////////////////////////////////////////////////////////////// +// When an "error" occurss - e.g. database cannot be found, file cannot be +// written, invalid argument passed into a function etc. a message is +// written to stderr that should end up in the viewer log +// TODO: Set the verbosity using the usual Viewer mechanism +void llDiskCache::printError(const std::string funcname, + const std::string desc, + bool is_sqlite_err) +{ + std::ostringstream err_msg; + + err_msg << "llDiskCache error in "; + err_msg << __FUNCDNAME__ << "(...) "; + err_msg << desc; + + if (is_sqlite_err) + { + err_msg << " - "; + err_msg << std::string(sqlite3_errmsg(mDb)); + } + + // TODO: set via viewer verbosity level + const int verbosity = 1; + if (verbosity > 0) + { + std::cerr << err_msg.str() << std::endl; + } +} + +/////////////////////////////////////////////////////////////////////////////// +// wrapper for SQLite code to begin an SQL transaction - not used yet but +// it will be eventually +bool llDiskCache::beginTransaction() +{ + const std::string stmt("BEGIN TRANSACTION"); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// wrapper for SQLite code to end an SQL transaction - not used yet but +// it will be eventually +bool llDiskCache::endTransaction() +{ + const std::string stmt("COMMIT"); + if (! sqliteExec(stmt, __FUNCDNAME__ )) + { + return false; + } + + return true; +} + +/////////////////////////////////////////////////////////////////////////////// +// Build a unique filename that will be used to store the actual file +// on disk (as opposed to the meta data in the database) +// TODO: I think this needs more work once we move it to the viewer +// and espcially to make it cross platform - see 'std::tmpnam' +// depreciation comments in compiler output for example +std::string llDiskCache::makeUniqueFilename() +{ + // TODO: replace with boost - this is marked as deprecated? + std::string base = std::tmpnam(nullptr); + + // C++11 random number generation!!! + static std::random_device dev; + static std::mt19937 rng(dev()); + std::uniform_int_distribution dist(100000, 999999); + + // currently the tmp filename from std::tmpnam() on macOS + // is of the form `/tmp/foo/bar.12345 and the following code + // strips all the preceding dirs - we likely want a more + // robust (and cross platform solution) when we move to the + // viewer code + std::size_t found = base.rfind(systemSeparator()); + if (found != std::string::npos && found < base.size() - 2) + { + base = base.substr(found + 1); + } + else + { + base = ""; + } + + // we mix in a random number for some more entropy.. + // (i know, i know...) + std::ostringstream unique_filename; + unique_filename << base; + unique_filename << "."; + unique_filename << dist(rng); + + return unique_filename.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// Return system file/path separator - likely replaced by the version +// in the viewer +const std::string llDiskCache::systemSeparator() +{ +// TODO: replace in viewer with relevant call +#ifdef _WIN32 + return "\\"; +#else + return "/"; +#endif +} + +/////////////////////////////////////////////////////////////////////////////// +// Given a filename, compose a full path based on the path name passed +// in when the database was opened and the separator in play. +const std::string llDiskCache::makeFullPath(const std::string filename) +{ + std::string pathname = mDataStorePath + systemSeparator() + filename; + + return pathname; +} + +/////////////////////////////////////////////////////////////////////////////// +// +const std::string llDiskCache::sqlComposeCreateTable() +{ + std::ostringstream stmt; + stmt << "CREATE TABLE IF NOT EXISTS "; + stmt << mTableName; + stmt << "("; + stmt << mIdFieldName; + stmt << " TEXT PRIMARY KEY, "; + stmt << mFilenameFieldName; + stmt << " TEXT, "; + stmt << mFilesizeFieldName; + stmt << " INTEGER DEFAULT 0, "; + stmt << mInsertionDateTimeFieldName; + stmt << " TEXT, "; + stmt << mLastAccessDateTimeFieldName; + stmt << " TEXT, "; + stmt << mAccessCountFieldName; + stmt << " INTEGER DEFAULT 0"; + stmt << ")"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// +const std::string llDiskCache::sqlComposeExists(const std::string id) +{ + std::ostringstream stmt; + stmt << "SELECT COUNT(*) FROM "; + stmt << mTableName; + stmt << " WHERE "; + stmt << mIdFieldName; + stmt << "='"; + stmt << id; + stmt << "'"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement to write an entry to the DB +// Saves id, filename (generated), file size and create/last access date +const std::string llDiskCache::sqlComposePut(const std::string id, + const std::string filename, + int binary_data_size) +{ + std::ostringstream stmt; + stmt << "REPLACE INTO "; + stmt << mTableName; + stmt << "("; + stmt << mIdFieldName; + stmt << ","; + stmt << mFilenameFieldName; + stmt << ","; + stmt << mFilesizeFieldName; + stmt << ","; + stmt << mInsertionDateTimeFieldName; + stmt << ","; + stmt << mLastAccessDateTimeFieldName; + stmt << ") "; + stmt << "VALUES("; + stmt << "'"; + stmt << id; + stmt << "', "; + stmt << "'"; + stmt << filename; + stmt << "', "; + stmt << binary_data_size; + stmt << ", "; + stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; + stmt << ", "; + stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; + stmt << ")"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement used in get() to look up the filename and file size +const std::string llDiskCache::sqlComposeGetSelect(const std::string id) +{ + std::ostringstream stmt; + stmt << "SELECT "; + stmt << mFilenameFieldName; + stmt << ", "; + stmt << mFilesizeFieldName; + stmt << " FROM "; + stmt << mTableName; + stmt << " WHERE "; + stmt << mIdFieldName; + stmt << "='"; + stmt << id; + stmt << "'"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement to update the date/time of last access as well as the +// count of number of times the file has been accessed. +// Note: the more accurate representation of date/time is used to +// ensure ms accuracy vs the standard INTEGER days since epoch approach +const std::string llDiskCache::sqlComposeGetUpdate(const std::string id) +{ + std::ostringstream stmt; + stmt << "UPDATE "; + stmt << mTableName; + stmt << " SET "; + stmt << mAccessCountFieldName; + stmt << "="; + stmt << mAccessCountFieldName; + stmt << "+1"; + stmt << ", "; + stmt << mLastAccessDateTimeFieldName; + stmt << "="; + stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; + stmt << " WHERE "; + stmt << mIdFieldName; + stmt << "='"; + stmt << id; + stmt << "'"; + +#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +#endif + + return stmt.str(); +} + +/////////////////////////////////////////////////////////////////////////////// +// SQL statement to remove items from the database that are older +// than the newest num_elements entries +const std::string llDiskCache::sqlComposePurge(int num_entries) +{ + std::ostringstream stmt; + stmt << "DELETE FROM "; + stmt << mTableName; + stmt << " WHERE "; + stmt << mLastAccessDateTimeFieldName; + stmt << " NOT IN "; + stmt << "("; + stmt << "SELECT "; + stmt << mLastAccessDateTimeFieldName; + stmt << " FROM "; + stmt << mTableName; + stmt << " ORDER BY "; + stmt << mLastAccessDateTimeFieldName; + stmt << " DESC"; + stmt << " LIMIT "; + stmt << num_entries; + stmt << ")"; + +//#ifdef SHOW_STATEMENTS + std::cout << stmt.str() << std::endl; +//#endif + + return stmt.str(); +} diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h new file mode 100644 index 0000000000..39b8f7ef48 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.h @@ -0,0 +1,130 @@ +/** + * @file lldiskcache.h + * @brief Declaration SQLite meta data / file storage API + * @brief Declaration of the generic disk cache interface + as well the SQLite header/implementation. + * @Note Eventually, this component might split into an interface + * file and multiple implemtations but for now, this is the + * only one so I think it's okay to combine everything. + * + * $LicenseInfo:firstyear=2009&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2020, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLDISKCACHE +#define _LLDISKCACHE + +#include +#include + +#include "sqlite3.h" + +// Enable this line to display each SQL statement when it is executed +// It can lead to a lot of spam but useful for debugging +//#define SHOW_STATEMENTS + +// I toyed with the idea of a variety of approaches and thought have +// an abstract base class with which to hang different implementations +// off was a good idea but I think we settled on a single approach +// so I will likely remove this level of indirection if not other +// interesting implementation ideas present themselves. +class llDiskCacheBase +{ + public: + llDiskCacheBase() {}; + virtual ~llDiskCacheBase() {}; + virtual bool open(const std::string db_folder, + const std::string db_filename) = 0; + virtual bool exists(const std::string id, bool& exists) = 0; + virtual bool put(const std::string id, + char* binary_data, + int binary_data_size) = 0; + virtual const bool get(const std::string id, + char** mem_buffer, + int& mem_buffer_size) = 0; + virtual bool purge(int num_entries) = 0; + virtual void close() = 0; + virtual const std::string dbVersion() = 0; + virtual const std::string getFilenameById(const std::string id) = 0; + virtual const int getAccessCountById(const std::string id) = 0; + virtual void getNumEntries(int& num_entries, int& max_entries) = 0; +}; + +// implementation for the SQLite/disk blended case +// see source file for detailed comments +class llDiskCache : + public llDiskCacheBase +{ + public: + llDiskCache(); + virtual ~llDiskCache(); + virtual bool open(const std::string db_folder, + const std::string db_filename) override; + virtual bool exists(const std::string id, bool& exists) override; + virtual bool put(const std::string id, + char* binary_data, + int binary_data_size) override; + virtual const bool get(const std::string id, + char** mem_buffer, + int& mem_buffer_size) override; + virtual bool purge(int num_entries) override; + virtual void close() override; + virtual const std::string dbVersion() override; + virtual const std::string getFilenameById(const std::string id) override; + virtual const int getAccessCountById(const std::string id) override; + virtual void getNumEntries(int& num_entries, int& max_entries) override; + + private: + sqlite3* mDb; + std::string mDataStorePath; + const std::string mTableName = "lldiskcache"; + const std::string mIdFieldName = "id"; + const std::string mFilenameFieldName = "filename"; + const std::string mFilesizeFieldName = "filesize"; + const std::string mInsertionDateTimeFieldName = "insert_datetime"; + const std::string mLastAccessDateTimeFieldName = "last_access_datetime"; + const std::string mAccessCountFieldName = "access_count"; + + private: + void printError(const std::string funcname, + const std::string desc, + bool is_sqlite_err); + bool beginTransaction(); + bool endTransaction(); + std::string makeUniqueFilename(); + const std::string systemSeparator(); + const std::string makeFullPath(const std::string filename); + bool sqliteExec(const std::string stmt, + const std::string funcname); + sqlite3_stmt* sqlitePrepareStep(const std::string funcname, + const std::string stmt, + int sqlite_success_condition); + const std::string sqlComposeCreateTable(); + const std::string sqlComposeExists(const std::string id); + const std::string sqlComposePut(const std::string id, + const std::string filename, + int binary_data_size); + const std::string sqlComposeGetSelect(const std::string id); + const std::string sqlComposeGetUpdate(const std::string id); + const std::string sqlComposePurge(int num_entries); +}; + +#endif // _LLDISKCACHE diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index af93049e07..f0037c9a22 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -1,6 +1,9 @@ /** - * @file lldiskcache.cpp - * @brief Implementation of virtual file + * @file filesystem.h + * @brief Simulate local file system operations. + * @Note The initial implementation does actually use standard C++ + * file operations but eventually, there will be another + * layer that caches and manages file meta data too. * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code @@ -26,25 +29,21 @@ #include "linden_common.h" -#include "lldiskcache.h" - -#include "llerror.h" -#include "llthread.h" -#include "lltimer.h" +#include "lldir.h" +#include "llfilesystem.h" #include "llfasttimer.h" -#include "llmemory.h" +#include "lldiskcache.h" #include -#include "lldir.h" -const S32 LLDiskCache::READ = 0x00000001; -const S32 LLDiskCache::WRITE = 0x00000002; -const S32 LLDiskCache::READ_WRITE = 0x00000003; // LLDiskCache::READ & LLDiskCache::WRITE -const S32 LLDiskCache::APPEND = 0x00000006; // 0x00000004 & LLDiskCache::WRITE +const S32 LLFileSystem::READ = 0x00000001; +const S32 LLFileSystem::WRITE = 0x00000002; +const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFileSystem::WRITE +const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); -LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) +LLFileSystem::LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) { mFileType = file_type; mFileID = file_id; @@ -54,7 +53,7 @@ LLDiskCache::LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_ty mMode = mode; } -LLDiskCache::~LLDiskCache() +LLFileSystem::~LLFileSystem() { } @@ -124,7 +123,7 @@ const std::string idToFilepath(const std::string id, LLAssetType::EType at) } // static -bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) +bool LLFileSystem::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); @@ -140,7 +139,7 @@ bool LLDiskCache::getExists(const LLUUID &file_id, const LLAssetType::EType file } // static -bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) +bool LLFileSystem::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); @@ -152,7 +151,7 @@ bool LLDiskCache::removeFile(const LLUUID &file_id, const LLAssetType::EType fil } // static -bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, +bool LLFileSystem::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, const LLUUID &new_file_id, const LLAssetType::EType new_file_type) { std::string old_id_str; @@ -175,7 +174,7 @@ bool LLDiskCache::renameFile(const LLUUID &old_file_id, const LLAssetType::EType } // static -S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) +S32 LLFileSystem::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); @@ -192,7 +191,7 @@ S32 LLDiskCache::getFileSize(const LLUUID &file_id, const LLAssetType::EType fil return file_size; } -BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) +BOOL LLFileSystem::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) { BOOL success = TRUE; @@ -232,7 +231,7 @@ BOOL LLDiskCache::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) return success; } -BOOL LLDiskCache::isReadComplete() +BOOL LLFileSystem::isReadComplete() { if (mReadComplete) { @@ -242,17 +241,17 @@ BOOL LLDiskCache::isReadComplete() return FALSE; } -S32 LLDiskCache::getLastBytesRead() +S32 LLFileSystem::getLastBytesRead() { return mBytesRead; } -BOOL LLDiskCache::eof() +BOOL LLFileSystem::eof() { return mPosition >= getSize(); } -BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) +BOOL LLFileSystem::write(const U8 *buffer, S32 bytes) { std::string id_str; mFileID.toString(id_str); @@ -287,14 +286,14 @@ BOOL LLDiskCache::write(const U8 *buffer, S32 bytes) } //static -BOOL LLDiskCache::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) +BOOL LLFileSystem::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) { - LLDiskCache file(uuid, type, LLDiskCache::WRITE); + LLFileSystem file(uuid, type, LLFileSystem::WRITE); file.setMaxSize(bytes); return file.write(buffer, bytes); } -BOOL LLDiskCache::seek(S32 offset, S32 origin) +BOOL LLFileSystem::seek(S32 offset, S32 origin) { if (-1 == origin) { @@ -324,32 +323,32 @@ BOOL LLDiskCache::seek(S32 offset, S32 origin) return TRUE; } -S32 LLDiskCache::tell() const +S32 LLFileSystem::tell() const { return mPosition; } -S32 LLDiskCache::getSize() +S32 LLFileSystem::getSize() { - return LLDiskCache::getFileSize(mFileID, mFileType); + return LLFileSystem::getFileSize(mFileID, mFileType); } -S32 LLDiskCache::getMaxSize() +S32 LLFileSystem::getMaxSize() { // offer up a huge size since we don't care what the max is return INT_MAX; } -BOOL LLDiskCache::setMaxSize(S32 size) +BOOL LLFileSystem::setMaxSize(S32 size) { // we don't care what the max size is so we do nothing // and return true to indicate all was okay return TRUE; } -BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type) +BOOL LLFileSystem::rename(const LLUUID &new_id, const LLAssetType::EType new_type) { - LLDiskCache::renameFile(mFileID, mFileType, new_id, new_type); + LLFileSystem::renameFile(mFileID, mFileType, new_id, new_type); mFileID = new_id; mFileType = new_type; @@ -357,31 +356,31 @@ BOOL LLDiskCache::rename(const LLUUID &new_id, const LLAssetType::EType new_type return TRUE; } -BOOL LLDiskCache::remove() +BOOL LLFileSystem::remove() { - LLDiskCache::removeFile(mFileID, mFileType); + LLFileSystem::removeFile(mFileID, mFileType); return TRUE; } // static -void LLDiskCache::initClass() +void LLFileSystem::initClass() { } // static -void LLDiskCache::cleanupClass() +void LLFileSystem::cleanupClass() { } -bool LLDiskCache::isLocked() +bool LLFileSystem::isLocked() { // I don't think we care about this test since there is no locking // TODO: remove this function and calling sites? return FALSE; } -void LLDiskCache::waitForLock() +void LLFileSystem::waitForLock() { // TODO: remove this function and calling sites? } diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h index 7ad06a8689..9cddef74a7 100644 --- a/indra/llfilesystem/llfilesystem.h +++ b/indra/llfilesystem/llfilesystem.h @@ -1,6 +1,9 @@ /** - * @file lldiskcacke.h - * @brief Definition of virtual file + * @file filesystem.h + * @brief Simulate local file system operations. + * @Note The initial implementation does actually use standard C++ + * file operations but eventually, there will be another + * layer that caches and manages file meta data too. * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code @@ -24,17 +27,17 @@ * $/LicenseInfo$ */ -#ifndef LL_LLDISKCACHE_H -#define LL_LLDISKCACHE_H +#ifndef LL_FILESYSTEM_H +#define LL_FILESYSTEM_H #include "lluuid.h" #include "llassettype.h" -class LLDiskCache +class LLFileSystem { public: - LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLDiskCache::READ); - ~LLDiskCache(); + LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLFileSystem::READ); + ~LLFileSystem(); BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ BOOL isReadComplete(); @@ -79,4 +82,4 @@ protected: S32 mBytesRead; }; -#endif // LL_LLDISKCACHE_H +#endif // LL_FILESYSTEM_H diff --git a/indra/llmessage/llassetstorage.cpp b/indra/llmessage/llassetstorage.cpp index 31c1edd75e..f38a5e663e 100644 --- a/indra/llmessage/llassetstorage.cpp +++ b/indra/llmessage/llassetstorage.cpp @@ -42,7 +42,7 @@ // this library includes #include "message.h" #include "llxfermanager.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "lldbstrings.h" #include "lltransfersourceasset.h" @@ -438,7 +438,7 @@ void LLAssetStorage::_cleanupRequests(BOOL all, S32 error) BOOL LLAssetStorage::hasLocalAsset(const LLUUID &uuid, const LLAssetType::EType type) { - return LLDiskCache::getExists(uuid, type); + return LLFileSystem::getExists(uuid, type); } bool LLAssetStorage::findInCacheAndInvokeCallback(const LLUUID& uuid, LLAssetType::EType type, @@ -450,10 +450,10 @@ bool LLAssetStorage::findInCacheAndInvokeCallback(const LLUUID& uuid, LLAssetTyp llassert(callback != NULL); } - BOOL exists = LLDiskCache::getExists(uuid, type); + BOOL exists = LLFileSystem::getExists(uuid, type); if (exists) { - LLDiskCache file(uuid, type); + LLFileSystem file(uuid, type); U32 size = file.getSize(); if (size > 0) { @@ -523,8 +523,8 @@ void LLAssetStorage::getAssetData(const LLUUID uuid, return; } - BOOL exists = LLDiskCache::getExists(uuid, type); - LLDiskCache file(uuid, type); + BOOL exists = LLFileSystem::getExists(uuid, type); + LLFileSystem file(uuid, type); U32 size = exists ? file.getSize() : 0; if (size > 0) @@ -664,7 +664,7 @@ void LLAssetStorage::downloadCompleteCallback( if (LL_ERR_NOERR == result) { // we might have gotten a zero-size file - LLDiskCache vfile(callback_id, callback_type); + LLFileSystem vfile(callback_id, callback_type); if (vfile.getSize() <= 0) { LL_WARNS("AssetStorage") << "downloadCompleteCallback has non-existent or zero-size asset " << callback_id << LL_ENDL; @@ -724,8 +724,8 @@ void LLAssetStorage::getEstateAsset( return; } - BOOL exists = LLDiskCache::getExists(asset_id, atype); - LLDiskCache file(asset_id, atype); + BOOL exists = LLFileSystem::getExists(asset_id, atype); + LLFileSystem file(asset_id, atype); U32 size = exists ? file.getSize() : 0; if (size > 0) @@ -818,7 +818,7 @@ void LLAssetStorage::downloadEstateAssetCompleteCallback( if (LL_ERR_NOERR == result) { // we might have gotten a zero-size file - LLDiskCache vfile(req->getUUID(), req->getAType()); + LLFileSystem vfile(req->getUUID(), req->getAType()); if (vfile.getSize() <= 0) { LL_WARNS("AssetStorage") << "downloadCompleteCallback has non-existent or zero-size asset!" << LL_ENDL; @@ -860,8 +860,8 @@ void LLAssetStorage::getInvItemAsset( return; } - exists = LLDiskCache::getExists(asset_id, atype); - LLDiskCache file(asset_id, atype); + exists = LLFileSystem::getExists(asset_id, atype); + LLFileSystem file(asset_id, atype); size = exists ? file.getSize() : 0; if(exists && size < 1) { @@ -961,7 +961,7 @@ void LLAssetStorage::downloadInvItemCompleteCallback( if (LL_ERR_NOERR == result) { // we might have gotten a zero-size file - LLDiskCache vfile(req->getUUID(), req->getType()); + LLFileSystem vfile(req->getUUID(), req->getType()); if (vfile.getSize() <= 0) { LL_WARNS("AssetStorage") << "downloadCompleteCallback has non-existent or zero-size asset!" << LL_ENDL; @@ -1396,7 +1396,7 @@ void LLAssetStorage::legacyGetDataCallback(const LLUUID &uuid, if ( !status && !toxic ) { - LLDiskCache file(uuid, type); + LLFileSystem file(uuid, type); std::string uuid_str; diff --git a/indra/llmessage/llcorehttputil.cpp b/indra/llmessage/llcorehttputil.cpp index 376558400c..7031f1aa8c 100644 --- a/indra/llmessage/llcorehttputil.cpp +++ b/indra/llmessage/llcorehttputil.cpp @@ -37,7 +37,7 @@ #include "llsdserialize.h" #include "reader.h" // JSON #include "writer.h" // JSON -#include "lldiskcache.h" +#include "llfilesystem.h" #include "message.h" // for getting the port @@ -784,7 +784,7 @@ LLSD HttpCoroutineAdapter::postFileAndSuspend(LLCore::HttpRequest::ptr_t request // scoping for our streams so that they go away when we no longer need them. { LLCore::BufferArrayStream outs(fileData.get()); - LLDiskCache vfile(assetId, assetType, LLDiskCache::READ); + LLFileSystem vfile(assetId, assetType, LLFileSystem::READ); S32 fileSize = vfile.getSize(); U8* fileBuffer; diff --git a/indra/llmessage/lltransfersourceasset.cpp b/indra/llmessage/lltransfersourceasset.cpp index 7b00a95b00..027283232d 100644 --- a/indra/llmessage/lltransfersourceasset.cpp +++ b/indra/llmessage/lltransfersourceasset.cpp @@ -32,7 +32,7 @@ #include "message.h" #include "lldatapacker.h" #include "lldir.h" -#include "lldiskcache.h" +#include "llfilesystem.h" LLTransferSourceAsset::LLTransferSourceAsset(const LLUUID &request_id, const F32 priority) : LLTransferSource(LLTST_ASSET, request_id, priority), @@ -99,7 +99,7 @@ LLTSCode LLTransferSourceAsset::dataCallback(const S32 packet_id, return LLTS_SKIP; } - LLDiskCache vf(mParams.getAssetID(), mParams.getAssetType(), LLDiskCache::READ); + LLFileSystem vf(mParams.getAssetID(), mParams.getAssetType(), LLFileSystem::READ); if (!vf.getSize()) { @@ -198,7 +198,7 @@ void LLTransferSourceAsset::responderCallback(const LLUUID& uuid, LLAssetType::E if (LL_ERR_NOERR == result) { // Everything's OK. - LLDiskCache vf(uuid, type, LLDiskCache::READ); + LLFileSystem vf(uuid, type, LLFileSystem::READ); tsap->mSize = vf.getSize(); status = LLTS_OK; } diff --git a/indra/llmessage/lltransfersourceasset.h b/indra/llmessage/lltransfersourceasset.h index d9055202ec..585e683cb3 100644 --- a/indra/llmessage/lltransfersourceasset.h +++ b/indra/llmessage/lltransfersourceasset.h @@ -30,7 +30,7 @@ #include "lltransfermanager.h" #include "llassetstorage.h" -class LLDiskCache; +class LLFileSystem; class LLTransferSourceParamsAsset : public LLTransferSourceParams { diff --git a/indra/llmessage/lltransfertargetvfile.cpp b/indra/llmessage/lltransfertargetvfile.cpp index f4a5e71d08..471d687d67 100644 --- a/indra/llmessage/lltransfertargetvfile.cpp +++ b/indra/llmessage/lltransfertargetvfile.cpp @@ -30,7 +30,7 @@ #include "lldatapacker.h" #include "llerror.h" -#include "lldiskcache.h" +#include "llfilesystem.h" //static void LLTransferTargetVFile::updateQueue(bool shutdown) @@ -138,7 +138,7 @@ LLTSCode LLTransferTargetVFile::dataCallback(const S32 packet_id, U8 *in_datap, //LL_INFOS() << "LLTransferTargetFile::dataCallback" << LL_ENDL; //LL_INFOS() << "Packet: " << packet_id << LL_ENDL; - LLDiskCache vf(mTempID, mParams.getAssetType(), LLDiskCache::APPEND); + LLFileSystem vf(mTempID, mParams.getAssetType(), LLFileSystem::APPEND); if (mNeedsCreate) { vf.setMaxSize(mSize); @@ -176,7 +176,7 @@ void LLTransferTargetVFile::completionCallback(const LLTSCode status) case LLTS_DONE: if (!mNeedsCreate) { - LLDiskCache file(mTempID, mParams.getAssetType(), LLDiskCache::WRITE); + LLFileSystem file(mTempID, mParams.getAssetType(), LLFileSystem::WRITE); if (!file.rename(mParams.getAssetID(), mParams.getAssetType())) { LL_ERRS() << "LLTransferTargetVFile: rename failed" << LL_ENDL; @@ -195,7 +195,7 @@ void LLTransferTargetVFile::completionCallback(const LLTSCode status) { // We're aborting this transfer, we don't want to keep this file. LL_WARNS() << "Aborting vfile transfer for " << mParams.getAssetID() << LL_ENDL; - LLDiskCache vf(mTempID, mParams.getAssetType(), LLDiskCache::APPEND); + LLFileSystem vf(mTempID, mParams.getAssetType(), LLFileSystem::APPEND); vf.remove(); } break; diff --git a/indra/llmessage/lltransfertargetvfile.h b/indra/llmessage/lltransfertargetvfile.h index 4c1bfe22c5..39a9125f1b 100644 --- a/indra/llmessage/lltransfertargetvfile.h +++ b/indra/llmessage/lltransfertargetvfile.h @@ -29,9 +29,9 @@ #include "lltransfermanager.h" #include "llassetstorage.h" -#include "lldiskcache.h" +#include "llfilesystem.h" -class LLDiskCache; +class LLFileSystem; // Lame, an S32 for now until I figure out the deal with how we want to do // error codes. diff --git a/indra/llmessage/llxfer_vfile.cpp b/indra/llmessage/llxfer_vfile.cpp index 95629d5fea..9de9ed379b 100644 --- a/indra/llmessage/llxfer_vfile.cpp +++ b/indra/llmessage/llxfer_vfile.cpp @@ -30,7 +30,7 @@ #include "lluuid.h" #include "llerror.h" #include "llmath.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "lldir.h" // size of chunks read from/written to disk @@ -79,9 +79,9 @@ void LLXfer_VFile::cleanup () if (mTempID.notNull() && mDeleteTempFile) { - if (LLDiskCache::getExists(mTempID, mType)) + if (LLFileSystem::getExists(mTempID, mType)) { - LLDiskCache file(mTempID, mType, LLDiskCache::WRITE); + LLFileSystem file(mTempID, mType, LLFileSystem::WRITE); file.remove(); } else @@ -187,9 +187,9 @@ S32 LLXfer_VFile::startSend (U64 xfer_id, const LLHost &remote_host) delete mVFile; mVFile = NULL; - if(LLDiskCache::getExists(mLocalID, mType)) + if(LLFileSystem::getExists(mLocalID, mType)) { - mVFile = new LLDiskCache(mLocalID, mType, LLDiskCache::READ); + mVFile = new LLFileSystem(mLocalID, mType, LLFileSystem::READ); if (mVFile->getSize() <= 0) { @@ -235,9 +235,9 @@ S32 LLXfer_VFile::reopenFileHandle() if (mVFile == NULL) { - if (LLDiskCache::getExists(mLocalID, mType)) + if (LLFileSystem::getExists(mLocalID, mType)) { - mVFile = new LLDiskCache(mLocalID, mType, LLDiskCache::READ); + mVFile = new LLFileSystem(mLocalID, mType, LLFileSystem::READ); } else { @@ -260,7 +260,7 @@ void LLXfer_VFile::setXferSize (S32 xfer_size) // It would be nice if LLXFers could tell which end of the pipe they were if (! mVFile) { - LLDiskCache file(mTempID, mType, LLDiskCache::APPEND); + LLFileSystem file(mTempID, mType, LLFileSystem::APPEND); file.setMaxSize(xfer_size); } } @@ -315,7 +315,7 @@ S32 LLXfer_VFile::flush() S32 retval = 0; if (mBufferLength) { - LLDiskCache file(mTempID, mType, LLDiskCache::APPEND); + LLFileSystem file(mTempID, mType, LLFileSystem::APPEND); file.write((U8*)mBuffer, mBufferLength); @@ -335,9 +335,9 @@ S32 LLXfer_VFile::processEOF() if (!mCallbackResult) { - if (LLDiskCache::getExists(mTempID, mType)) + if (LLFileSystem::getExists(mTempID, mType)) { - LLDiskCache file(mTempID, mType, LLDiskCache::WRITE); + LLFileSystem file(mTempID, mType, LLFileSystem::WRITE); if (!file.rename(mLocalID, mType)) { LL_WARNS("Xfer") << "Cache rename of temp file failed: unable to rename " << mTempID << " to " << mLocalID << LL_ENDL; diff --git a/indra/llmessage/llxfer_vfile.h b/indra/llmessage/llxfer_vfile.h index d6ac6ff818..d82bab5f6c 100644 --- a/indra/llmessage/llxfer_vfile.h +++ b/indra/llmessage/llxfer_vfile.h @@ -30,7 +30,7 @@ #include "llxfer.h" #include "llassetstorage.h" -class LLDiskCache; +class LLFileSystem; class LLXfer_VFile : public LLXfer { @@ -40,7 +40,7 @@ class LLXfer_VFile : public LLXfer LLUUID mTempID; LLAssetType::EType mType; - LLDiskCache *mVFile; + LLFileSystem *mVFile; std::string mName; diff --git a/indra/llui/llviewereventrecorder.h b/indra/llui/llviewereventrecorder.h index fec0f9784f..94e66f5dc4 100644 --- a/indra/llui/llviewereventrecorder.h +++ b/indra/llui/llviewereventrecorder.h @@ -32,7 +32,6 @@ #include "lldir.h" #include "llsd.h" #include "llfile.h" -#include "lldiskcache.h" #include "lldate.h" #include "llsdserialize.h" #include "llkeyboard.h" diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 6181683b9e..12598d028c 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -114,7 +114,7 @@ #include "llprimitive.h" #include "llurlaction.h" #include "llurlentry.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llvolumemgr.h" #include "llxfermanager.h" #include "llphysicsextensions.h" @@ -1876,7 +1876,7 @@ bool LLAppViewer::cleanup() SUBSYSTEM_CLEANUP(LLFolderViewItem); LL_INFOS() << "Cleaning up disk cache" << LL_ENDL; - SUBSYSTEM_CLEANUP(LLDiskCache); + SUBSYSTEM_CLEANUP(LLFileSystem); LL_INFOS() << "Saving Data" << LL_ENDL; @@ -4160,7 +4160,7 @@ bool LLAppViewer::initCache() LLVOCache::getInstance()->initCache(LL_PATH_CACHE, gSavedSettings.getU32("CacheNumberOfRegionsForObjects"), getObjectCacheVersion()); - LLDiskCache::initClass(); + LLFileSystem::initClass(); return true; } diff --git a/indra/newview/llcompilequeue.cpp b/indra/newview/llcompilequeue.cpp index ad2f3b9f9a..5d010a6f1e 100644 --- a/indra/newview/llcompilequeue.cpp +++ b/indra/newview/llcompilequeue.cpp @@ -52,7 +52,7 @@ #include "lldir.h" #include "llnotificationsutil.h" #include "llviewerstats.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "lluictrlfactory.h" #include "lltrans.h" diff --git a/indra/newview/llfloaterauction.cpp b/indra/newview/llfloaterauction.cpp index 42bcb86454..1b9814883e 100644 --- a/indra/newview/llfloaterauction.cpp +++ b/indra/newview/llfloaterauction.cpp @@ -32,7 +32,7 @@ #include "llimagej2c.h" #include "llimagetga.h" #include "llparcel.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llwindow.h" #include "message.h" @@ -201,7 +201,7 @@ void LLFloaterAuction::onClickSnapshot(void* data) LLPointer tga = new LLImageTGA; tga->encode(raw); - LLDiskCache::writeFile(tga->getData(), tga->getDataSize(), self->mImageID, LLAssetType::AT_IMAGE_TGA); + LLFileSystem::writeFile(tga->getData(), tga->getDataSize(), self->mImageID, LLAssetType::AT_IMAGE_TGA); raw->biasedScaleToPowerOfTwo(LLViewerTexture::MAX_IMAGE_SIZE_DEFAULT); @@ -209,7 +209,7 @@ void LLFloaterAuction::onClickSnapshot(void* data) LLPointer j2c = new LLImageJ2C; j2c->encode(raw, 0.0f); - LLDiskCache::writeFile(j2c->getData(), j2c->getDataSize(), self->mImageID, LLAssetType::AT_TEXTURE); + LLFileSystem::writeFile(j2c->getData(), j2c->getDataSize(), self->mImageID, LLAssetType::AT_TEXTURE); self->mImage = LLViewerTextureManager::getLocalTexture((LLImageRaw*)raw, FALSE); gGL.getTexUnit(0)->bind(self->mImage); diff --git a/indra/newview/llfloaterbvhpreview.cpp b/indra/newview/llfloaterbvhpreview.cpp index 303b4836e4..08f3b577b4 100644 --- a/indra/newview/llfloaterbvhpreview.cpp +++ b/indra/newview/llfloaterbvhpreview.cpp @@ -32,7 +32,7 @@ #include "lldatapacker.h" #include "lldir.h" #include "llnotificationsutil.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llapr.h" #include "llstring.h" @@ -997,7 +997,7 @@ void LLFloaterBvhPreview::onBtnOK(void* userdata) LLDataPackerBinaryBuffer dp(buffer, file_size); if (motionp->serialize(dp)) { - LLDiskCache file(motionp->getID(), LLAssetType::AT_ANIMATION, LLDiskCache::APPEND); + LLFileSystem file(motionp->getID(), LLAssetType::AT_ANIMATION, LLFileSystem::APPEND); S32 size = dp.getCurrentSize(); file.setMaxSize(size); diff --git a/indra/newview/llfloatermodelpreview.cpp b/indra/newview/llfloatermodelpreview.cpp index 057c4d0d5c..0fa5f8e4df 100644 --- a/indra/newview/llfloatermodelpreview.cpp +++ b/indra/newview/llfloatermodelpreview.cpp @@ -77,7 +77,7 @@ #include "llspinctrl.h" #include "lltoggleablemenu.h" #include "lltrans.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llcallbacklist.h" #include "llviewerobjectlist.h" #include "llanimationstates.h" diff --git a/indra/newview/llfloaterregioninfo.cpp b/indra/newview/llfloaterregioninfo.cpp index d2ab15a9b4..0375c15467 100644 --- a/indra/newview/llfloaterregioninfo.cpp +++ b/indra/newview/llfloaterregioninfo.cpp @@ -36,7 +36,7 @@ #include "llglheaders.h" #include "llregionflags.h" #include "llstl.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llxfermanager.h" #include "indra_constants.h" #include "message.h" @@ -2239,7 +2239,7 @@ void LLPanelEstateCovenant::onLoadComplete(const LLUUID& asset_uuid, { if(0 == status) { - LLDiskCache file(asset_uuid, type, LLDiskCache::READ); + LLFileSystem file(asset_uuid, type, LLFileSystem::READ); S32 file_length = file.getSize(); diff --git a/indra/newview/llfloaterreporter.cpp b/indra/newview/llfloaterreporter.cpp index 97db905467..3ef80300ef 100644 --- a/indra/newview/llfloaterreporter.cpp +++ b/indra/newview/llfloaterreporter.cpp @@ -44,7 +44,7 @@ #include "llnotificationsutil.h" #include "llstring.h" #include "llsys.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "mean_collision_data.h" #include "message.h" #include "v3math.h" @@ -899,7 +899,7 @@ void LLFloaterReporter::takeScreenshot(bool use_prev_screenshot) mResourceDatap->mAssetInfo.setDescription("screenshot_descr"); // store in cache - LLDiskCache::writeFile(upload_data->getData(), + LLFileSystem::writeFile(upload_data->getData(), upload_data->getDataSize(), mResourceDatap->mAssetInfo.mUuid, mResourceDatap->mAssetInfo.mType); diff --git a/indra/newview/llfloatertos.cpp b/indra/newview/llfloatertos.cpp index 96da13915c..1aeb727172 100644 --- a/indra/newview/llfloatertos.cpp +++ b/indra/newview/llfloatertos.cpp @@ -40,7 +40,7 @@ #include "lltextbox.h" #include "llui.h" #include "lluictrlfactory.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "message.h" #include "llstartup.h" // login_alert_done #include "llcorehttputil.h" diff --git a/indra/newview/llgesturemgr.cpp b/indra/newview/llgesturemgr.cpp index 82feb891bc..9f2119281d 100644 --- a/indra/newview/llgesturemgr.cpp +++ b/indra/newview/llgesturemgr.cpp @@ -42,7 +42,7 @@ #include "llnotificationsutil.h" #include "llstl.h" #include "llstring.h" // todo: remove -#include "lldiskcache.h" +#include "llfilesystem.h" #include "message.h" // newview @@ -1055,7 +1055,7 @@ void LLGestureMgr::onLoadComplete(const LLUUID& asset_uuid, if (0 == status) { - LLDiskCache file(asset_uuid, type, LLDiskCache::READ); + LLFileSystem file(asset_uuid, type, LLFileSystem::READ); S32 size = file.getSize(); std::vector buffer(size+1); diff --git a/indra/newview/lllandmarklist.cpp b/indra/newview/lllandmarklist.cpp index 5661b2525b..747212ba55 100644 --- a/indra/newview/lllandmarklist.cpp +++ b/indra/newview/lllandmarklist.cpp @@ -33,7 +33,7 @@ #include "llappviewer.h" #include "llagent.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llviewerstats.h" // Globals @@ -122,7 +122,7 @@ void LLLandmarkList::processGetAssetReply( { if( status == 0 ) { - LLDiskCache file(uuid, type); + LLFileSystem file(uuid, type); S32 file_length = file.getSize(); std::vector buffer(file_length + 1); diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 011b187b58..a0d591dc47 100644 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -49,7 +49,7 @@ #include "llsdutil_math.h" #include "llsdserialize.h" #include "llthread.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llviewercontrol.h" #include "llviewerinventory.h" #include "llviewermenufile.h" @@ -1335,7 +1335,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) if (version <= MAX_MESH_VERSION && offset >= 0 && size > 0) { //check cache for mesh skin info - LLDiskCache file(mesh_id, LLAssetType::AT_MESH); + LLFileSystem file(mesh_id, LLAssetType::AT_MESH); if (file.getSize() >= offset+size) { U8* buffer = new(std::nothrow) U8[size]; @@ -1431,7 +1431,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) if (version <= MAX_MESH_VERSION && offset >= 0 && size > 0) { //check cache for mesh skin info - LLDiskCache file(mesh_id, LLAssetType::AT_MESH); + LLFileSystem file(mesh_id, LLAssetType::AT_MESH); if (file.getSize() >= offset+size) { U8* buffer = new(std::nothrow) U8[size]; @@ -1528,7 +1528,7 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) if (version <= MAX_MESH_VERSION && offset >= 0 && size > 0) { //check cache for mesh physics shape info - LLDiskCache file(mesh_id, LLAssetType::AT_MESH); + LLFileSystem file(mesh_id, LLAssetType::AT_MESH); if (file.getSize() >= offset+size) { LLMeshRepository::sCacheBytesRead += size; @@ -1633,7 +1633,7 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, bool c { //look for mesh in asset in cache - LLDiskCache file(mesh_params.getSculptID(), LLAssetType::AT_MESH); + LLFileSystem file(mesh_params.getSculptID(), LLAssetType::AT_MESH); S32 size = file.getSize(); @@ -1712,7 +1712,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, { //check cache for mesh asset - LLDiskCache file(mesh_id, LLAssetType::AT_MESH); + LLFileSystem file(mesh_id, LLAssetType::AT_MESH); if (file.getSize() >= offset+size) { U8* buffer = new(std::nothrow) U8[size]; @@ -3240,7 +3240,7 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * /* body */, S32 /* b // only allocate as much space in the cache as is needed for the local cache data_size = llmin(data_size, bytes); - LLDiskCache file(mesh_id, LLAssetType::AT_MESH, LLDiskCache::WRITE); + LLFileSystem file(mesh_id, LLAssetType::AT_MESH, LLFileSystem::WRITE); if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) { LLMeshRepository::sCacheBytesWritten += data_size; @@ -3312,7 +3312,7 @@ void LLMeshLODHandler::processData(LLCore::BufferArray * /* body */, S32 /* body if (result == MESH_OK) { // good fetch from sim, write to cache - LLDiskCache file(mMeshParams.getSculptID(), LLAssetType::AT_MESH, LLDiskCache::WRITE); + LLFileSystem file(mMeshParams.getSculptID(), LLAssetType::AT_MESH, LLFileSystem::WRITE); S32 offset = mOffset; S32 size = mRequestedBytes; @@ -3376,7 +3376,7 @@ void LLMeshSkinInfoHandler::processData(LLCore::BufferArray * /* body */, S32 /* && gMeshRepo.mThread->skinInfoReceived(mMeshID, data, data_size)) { // good fetch from sim, write to cache - LLDiskCache file(mMeshID, LLAssetType::AT_MESH, LLDiskCache::WRITE); + LLFileSystem file(mMeshID, LLAssetType::AT_MESH, LLFileSystem::WRITE); S32 offset = mOffset; S32 size = mRequestedBytes; @@ -3424,7 +3424,7 @@ void LLMeshDecompositionHandler::processData(LLCore::BufferArray * /* body */, S && gMeshRepo.mThread->decompositionReceived(mMeshID, data, data_size)) { // good fetch from sim, write to cache - LLDiskCache file(mMeshID, LLAssetType::AT_MESH, LLDiskCache::WRITE); + LLFileSystem file(mMeshID, LLAssetType::AT_MESH, LLFileSystem::WRITE); S32 offset = mOffset; S32 size = mRequestedBytes; @@ -3471,7 +3471,7 @@ void LLMeshPhysicsShapeHandler::processData(LLCore::BufferArray * /* body */, S3 && gMeshRepo.mThread->physicsShapeReceived(mMeshID, data, data_size) == MESH_OK) { // good fetch from sim, write to cache for caching - LLDiskCache file(mMeshID, LLAssetType::AT_MESH, LLDiskCache::WRITE); + LLFileSystem file(mMeshID, LLAssetType::AT_MESH, LLFileSystem::WRITE); S32 offset = mOffset; S32 size = mRequestedBytes; diff --git a/indra/newview/lloutfitgallery.cpp b/indra/newview/lloutfitgallery.cpp index ac2d449621..90f6d23a61 100644 --- a/indra/newview/lloutfitgallery.cpp +++ b/indra/newview/lloutfitgallery.cpp @@ -32,7 +32,7 @@ // llcommon #include "llcommonutils.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llaccordionctrltab.h" #include "llappearancemgr.h" diff --git a/indra/newview/llpostcard.cpp b/indra/newview/llpostcard.cpp index 1fd57ef555..071fc31d27 100644 --- a/indra/newview/llpostcard.cpp +++ b/indra/newview/llpostcard.cpp @@ -28,7 +28,7 @@ #include "llpostcard.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llviewerregion.h" #include "message.h" diff --git a/indra/newview/llpreviewgesture.cpp b/indra/newview/llpreviewgesture.cpp index c7f8f790f4..371153aac3 100644 --- a/indra/newview/llpreviewgesture.cpp +++ b/indra/newview/llpreviewgesture.cpp @@ -47,7 +47,7 @@ #include "llradiogroup.h" #include "llresmgr.h" #include "lltrans.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llviewerobjectlist.h" #include "llviewerregion.h" #include "llviewerstats.h" @@ -852,7 +852,7 @@ void LLPreviewGesture::onLoadComplete(const LLUUID& asset_uuid, { if (0 == status) { - LLDiskCache file(asset_uuid, type, LLDiskCache::READ); + LLFileSystem file(asset_uuid, type, LLFileSystem::READ); S32 size = file.getSize(); std::vector buffer(size+1); @@ -1137,7 +1137,7 @@ void LLPreviewGesture::saveIfNeeded() tid.generate(); assetId = tid.makeAssetID(gAgent.getSecureSessionID()); - LLDiskCache file(assetId, LLAssetType::AT_GESTURE, LLDiskCache::APPEND); + LLFileSystem file(assetId, LLAssetType::AT_GESTURE, LLFileSystem::APPEND); S32 size = dp.getCurrentSize(); file.setMaxSize(size); diff --git a/indra/newview/llpreviewnotecard.cpp b/indra/newview/llpreviewnotecard.cpp index 0b21ff5047..0bccf1d06f 100644 --- a/indra/newview/llpreviewnotecard.cpp +++ b/indra/newview/llpreviewnotecard.cpp @@ -46,7 +46,7 @@ #include "llselectmgr.h" #include "lltrans.h" #include "llviewertexteditor.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llviewerinventory.h" #include "llviewerobject.h" #include "llviewerobjectlist.h" @@ -337,7 +337,7 @@ void LLPreviewNotecard::onLoadComplete(const LLUUID& asset_uuid, { if(0 == status) { - LLDiskCache file(asset_uuid, type, LLDiskCache::READ); + LLFileSystem file(asset_uuid, type, LLFileSystem::READ); S32 file_length = file.getSize(); @@ -452,7 +452,7 @@ void LLPreviewNotecard::finishInventoryUpload(LLUUID itemId, LLUUID newAssetId, // script actually changed the asset. if (nc->hasEmbeddedInventory()) { - LLDiskCache::removeFile(newAssetId, LLAssetType::AT_NOTECARD); + LLFileSystem::removeFile(newAssetId, LLAssetType::AT_NOTECARD); } if (newItemId.isNull()) { @@ -477,7 +477,7 @@ void LLPreviewNotecard::finishTaskUpload(LLUUID itemId, LLUUID newAssetId, LLUUI { if (nc->hasEmbeddedInventory()) { - LLDiskCache::removeFile(newAssetId, LLAssetType::AT_NOTECARD); + LLFileSystem::removeFile(newAssetId, LLAssetType::AT_NOTECARD); } nc->setAssetId(newAssetId); nc->refreshFromInventory(); @@ -556,7 +556,7 @@ bool LLPreviewNotecard::saveIfNeeded(LLInventoryItem* copyitem, bool sync) tid.generate(); asset_id = tid.makeAssetID(gAgent.getSecureSessionID()); - LLDiskCache file(asset_id, LLAssetType::AT_NOTECARD, LLDiskCache::APPEND); + LLFileSystem file(asset_id, LLAssetType::AT_NOTECARD, LLFileSystem::APPEND); LLSaveNotecardInfo* info = new LLSaveNotecardInfo(this, mItemUUID, mObjectUUID, diff --git a/indra/newview/llpreviewscript.cpp b/indra/newview/llpreviewscript.cpp index c2b687cf3b..eae6c28e35 100644 --- a/indra/newview/llpreviewscript.cpp +++ b/indra/newview/llpreviewscript.cpp @@ -51,7 +51,7 @@ #include "llsdserialize.h" #include "llslider.h" #include "lltooldraganddrop.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llagent.h" #include "llmenugl.h" @@ -1715,7 +1715,7 @@ void LLPreviewLSL::onLoadComplete(const LLUUID& asset_uuid, LLAssetType::EType t { if(0 == status) { - LLDiskCache file(asset_uuid, type); + LLFileSystem file(asset_uuid, type); S32 file_length = file.getSize(); std::vector buffer(file_length+1); @@ -2020,7 +2020,7 @@ void LLLiveLSLEditor::onLoadComplete(const LLUUID& asset_id, void LLLiveLSLEditor::loadScriptText(const LLUUID &uuid, LLAssetType::EType type) { - LLDiskCache file(uuid, type); + LLFileSystem file(uuid, type); S32 file_length = file.getSize(); std::vector buffer(file_length + 1); file.read((U8*)&buffer[0], file_length); diff --git a/indra/newview/llsettingsvo.cpp b/indra/newview/llsettingsvo.cpp index 62dc9f24bd..2db0fe6fa4 100644 --- a/indra/newview/llsettingsvo.cpp +++ b/indra/newview/llsettingsvo.cpp @@ -57,7 +57,7 @@ #include "llinventorymodel.h" #include "llassetstorage.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "lldrawpoolwater.h" #include @@ -303,7 +303,7 @@ void LLSettingsVOBase::onAssetDownloadComplete(const LLUUID &asset_id, S32 statu LLSettingsBase::ptr_t settings; if (!status) { - LLDiskCache file(asset_id, LLAssetType::AT_SETTINGS, LLDiskCache::READ); + LLFileSystem file(asset_id, LLAssetType::AT_SETTINGS, LLFileSystem::READ); S32 size = file.getSize(); std::string buffer(size + 1, '\0'); diff --git a/indra/newview/llsnapshotlivepreview.cpp b/indra/newview/llsnapshotlivepreview.cpp index 50523e981a..795563e295 100644 --- a/indra/newview/llsnapshotlivepreview.cpp +++ b/indra/newview/llsnapshotlivepreview.cpp @@ -31,6 +31,7 @@ #include "llagentbenefits.h" #include "llagentcamera.h" #include "llagentui.h" +#include "llfilesystem.h" #include "llcombobox.h" #include "llfloaterperms.h" #include "llfloaterreg.h" @@ -50,7 +51,6 @@ #include "llviewercontrol.h" #include "llviewermenufile.h" // upload_new_resource() #include "llviewerstats.h" -#include "lldiskcache.h" #include "llwindow.h" #include "llworld.h" #include @@ -1005,7 +1005,7 @@ void LLSnapshotLivePreview::saveTexture(BOOL outfit_snapshot, std::string name) if (formatted->encode(scaled, 0.0f)) { - LLDiskCache::writeFile(formatted->getData(), formatted->getDataSize(), new_asset_id, LLAssetType::AT_TEXTURE); + LLFileSystem::writeFile(formatted->getData(), formatted->getDataSize(), new_asset_id, LLAssetType::AT_TEXTURE); std::string pos_string; LLAgentUI::buildLocationString(pos_string, LLAgentUI::LOCATION_FORMAT_FULL); std::string who_took_it; diff --git a/indra/newview/llviewerassetstorage.cpp b/indra/newview/llviewerassetstorage.cpp index 3b86f3910d..df3ff1a3c7 100644 --- a/indra/newview/llviewerassetstorage.cpp +++ b/indra/newview/llviewerassetstorage.cpp @@ -28,7 +28,7 @@ #include "llviewerassetstorage.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "message.h" #include "llagent.h" @@ -152,13 +152,13 @@ void LLViewerAssetStorage::storeAssetData( if (mUpstreamHost.isOk()) { - if (LLDiskCache::getExists(asset_id, asset_type)) + if (LLFileSystem::getExists(asset_id, asset_type)) { // Pack data into this packet if we can fit it. U8 buffer[MTUBYTES]; buffer[0] = 0; - LLDiskCache vfile(asset_id, asset_type, LLDiskCache::READ); + LLFileSystem vfile(asset_id, asset_type, LLFileSystem::READ); S32 asset_size = vfile.getSize(); LLAssetRequest *req = new LLAssetRequest(asset_id, asset_type); @@ -180,7 +180,7 @@ void LLViewerAssetStorage::storeAssetData( else { // LLAssetStorage metric: Successful Request - S32 size = LLDiskCache::getFileSize(asset_id, asset_type); + S32 size = LLFileSystem::getFileSize(asset_id, asset_type); const char *message = "Added to upload queue"; reportMetric( asset_id, asset_type, LLStringUtil::null, LLUUID::null, size, MR_OKAY, __FILE__, __LINE__, message ); @@ -291,7 +291,7 @@ void LLViewerAssetStorage::storeAssetData( legacy->mUpCallback = callback; legacy->mUserData = user_data; - LLDiskCache file(asset_id, asset_type, LLDiskCache::WRITE); + LLFileSystem file(asset_id, asset_type, LLFileSystem::WRITE); file.setMaxSize(size); @@ -527,7 +527,7 @@ void LLViewerAssetStorage::assetRequestCoro( // case. LLUUID temp_id; temp_id.generate(); - LLDiskCache vf(temp_id, atype, LLDiskCache::WRITE); + LLFileSystem vf(temp_id, atype, LLFileSystem::WRITE); vf.setMaxSize(size); req->mBytesFetched = size; if (!vf.write(raw.data(),size)) diff --git a/indra/newview/llviewerassetstorage.h b/indra/newview/llviewerassetstorage.h index e65bdc1aea..972c89de34 100644 --- a/indra/newview/llviewerassetstorage.h +++ b/indra/newview/llviewerassetstorage.h @@ -30,7 +30,7 @@ #include "llassetstorage.h" #include "llcorehttputil.h" -class LLDiskCache; +class LLFileSystem; class LLViewerAssetRequest; diff --git a/indra/newview/llviewerassetupload.cpp b/indra/newview/llviewerassetupload.cpp index d62962514c..fb3ca69d5d 100644 --- a/indra/newview/llviewerassetupload.cpp +++ b/indra/newview/llviewerassetupload.cpp @@ -45,7 +45,7 @@ #include "llviewerassetupload.h" #include "llappviewer.h" #include "llviewerstats.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llgesturemgr.h" #include "llpreviewnotecard.h" #include "llpreviewgesture.h" @@ -472,7 +472,7 @@ LLSD LLNewFileResourceUploadInfo::exportTempFile() infile.open(filename, LL_APR_RB, NULL, &file_size); if (infile.getFileHandle()) { - LLDiskCache file(getAssetId(), assetType, LLDiskCache::WRITE); + LLFileSystem file(getAssetId(), assetType, LLFileSystem::WRITE); file.setMaxSize(file_size); @@ -565,7 +565,7 @@ LLSD LLBufferedAssetUploadInfo::prepareUpload() if (getAssetId().isNull()) generateNewAssetId(); - LLDiskCache file(getAssetId(), getAssetType(), LLDiskCache::APPEND); + LLFileSystem file(getAssetId(), getAssetType(), LLFileSystem::APPEND); S32 size = mContents.length() + 1; file.setMaxSize(size); @@ -597,7 +597,7 @@ LLUUID LLBufferedAssetUploadInfo::finishUpload(LLSD &result) if (mStoredToCache) { LLAssetType::EType assetType(getAssetType()); - LLDiskCache::renameFile(getAssetId(), assetType, newAssetId, assetType); + LLFileSystem::renameFile(getAssetId(), assetType, newAssetId, assetType); } if (mTaskUpload) diff --git a/indra/newview/llviewermenufile.cpp b/indra/newview/llviewermenufile.cpp index 6d4e12528d..b34f4b5016 100644 --- a/indra/newview/llviewermenufile.cpp +++ b/indra/newview/llviewermenufile.cpp @@ -53,7 +53,6 @@ #include "llviewercontrol.h" // gSavedSettings #include "llviewertexturelist.h" #include "lluictrlfactory.h" -#include "lldiskcache.h" #include "llviewerinventory.h" #include "llviewermenu.h" // gMenuHolder #include "llviewerparcelmgr.h" diff --git a/indra/newview/llviewermessage.cpp b/indra/newview/llviewermessage.cpp index 10ee92c130..4ab6fcc580 100644 --- a/indra/newview/llviewermessage.cpp +++ b/indra/newview/llviewermessage.cpp @@ -44,7 +44,7 @@ #include "llteleportflags.h" #include "lltoastnotifypanel.h" #include "lltransactionflags.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llxfermanager.h" #include "mean_collision_data.h" @@ -6816,7 +6816,7 @@ void onCovenantLoadComplete(const LLUUID& asset_uuid, std::string covenant_text; if(0 == status) { - LLDiskCache file(asset_uuid, type, LLDiskCache::READ); + LLFileSystem file(asset_uuid, type, LLFileSystem::READ); S32 file_length = file.getSize(); diff --git a/indra/newview/llviewerstats.cpp b/indra/newview/llviewerstats.cpp index 874982ed60..7ea702b0c9 100644 --- a/indra/newview/llviewerstats.cpp +++ b/indra/newview/llviewerstats.cpp @@ -33,7 +33,6 @@ #include "llfloaterreg.h" #include "llmemory.h" #include "lltimer.h" -#include "lldiskcache.h" #include "llappviewer.h" diff --git a/indra/newview/llviewertexlayer.cpp b/indra/newview/llviewertexlayer.cpp index 199f7f710a..4c2fbcf837 100644 --- a/indra/newview/llviewertexlayer.cpp +++ b/indra/newview/llviewertexlayer.cpp @@ -31,7 +31,6 @@ #include "llagent.h" #include "llimagej2c.h" #include "llnotificationsutil.h" -#include "lldiskcache.h" #include "llviewerregion.h" #include "llglslshader.h" #include "llvoavatarself.h" diff --git a/indra/newview/llviewertexture.cpp b/indra/newview/llviewertexture.cpp index f12ab59e2b..d7377a2983 100644 --- a/indra/newview/llviewertexture.cpp +++ b/indra/newview/llviewertexture.cpp @@ -39,7 +39,6 @@ #include "llimagej2c.h" #include "llimagetga.h" #include "llstl.h" -#include "lldiskcache.h" #include "message.h" #include "lltimer.h" diff --git a/indra/newview/llviewertexture.h b/indra/newview/llviewertexture.h index 57a2421065..07997e02a5 100644 --- a/indra/newview/llviewertexture.h +++ b/indra/newview/llviewertexture.h @@ -54,7 +54,7 @@ class LLTexturePipelineTester ; typedef void (*loaded_callback_func)( BOOL success, LLViewerFetchedTexture *src_vi, LLImageRaw* src, LLImageRaw* src_aux, S32 discard_level, BOOL final, void* userdata ); -class LLDiskCache; +class LLFileSystem; class LLMessageSystem; class LLViewerMediaImpl ; class LLVOVolume ; diff --git a/indra/newview/llviewertexturelist.cpp b/indra/newview/llviewertexturelist.cpp index e0727d51ba..38fccba169 100644 --- a/indra/newview/llviewertexturelist.cpp +++ b/indra/newview/llviewertexturelist.cpp @@ -41,7 +41,7 @@ #include "llsdserialize.h" #include "llsys.h" -#include "lldiskcache.h" +#include "llfilesystem.h" #include "llxmltree.h" #include "message.h" -- cgit v1.3 From 3092aa8aae496803707980eb456cddbb9960ef1c Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 6 Oct 2020 17:16:53 -0700 Subject: Add in the C++ filesystem based cache and clean up some indempotent functions in llfilesystem --- indra/llfilesystem/lldiskcache.cpp | 809 +++++------------------------- indra/llfilesystem/lldiskcache.h | 146 ++---- indra/llfilesystem/llfilesystem.cpp | 32 +- indra/llfilesystem/llfilesystem.h | 10 +- indra/llmessage/lltransfertargetvfile.cpp | 1 - indra/llmessage/llxfer_vfile.cpp | 1 - indra/newview/app_settings/settings.xml | 11 + indra/newview/llfloaterbvhpreview.cpp | 1 - indra/newview/llmeshrepository.cpp | 2 +- indra/newview/llpreviewgesture.cpp | 1 - indra/newview/llpreviewnotecard.cpp | 1 - indra/newview/llviewerassetstorage.cpp | 3 - indra/newview/llviewerassetupload.cpp | 3 - 13 files changed, 196 insertions(+), 825 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 72982db069..4b2ba0dd79 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -1,9 +1,6 @@ /** * @file lldiskcache.cpp - * @brief The SQLite based disk cache implementation. - * @Note Eventually, this component might split into an interface - * file and multiple implemtations but for now, this is the - * only one so I think it's okay to combine everything. + * @brief The disk cache implementation. * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code @@ -27,747 +24,167 @@ * $/LicenseInfo$ */ -#if (defined(LL_WINDOWS) || defined(LL_LINUX) || defined(LL_DARWIN)) #include "linden_common.h" -#endif +#include "lluuid.h" +#include "lldir.h" #include "lldiskcache.h" -#include -#include -#include -#include -#include - -/////////////////////////////////////////////////////////////////////////////// -// -llDiskCache::llDiskCache() : - mDataStorePath("") -{ -} +#include +#include -llDiskCache::~llDiskCache() -{ -} +#include +#include +#include -/////////////////////////////////////////////////////////////////////////////// -// Opens the database - typically done when the application starts -// and is complementary to close() which is called when the application -// is finisahed and exits. -// Pass in the folder and filename of the SQLite database you want to -// use or create (file doesn't have to exist but the folder must) -// Returns true or false and writes a message to console on error -bool llDiskCache::open(const std::string db_folder, const std::string db_filename) +LLDiskCache::LLDiskCache(const std::string cache_dir) : + mCacheDir(cache_dir), + mMaxSizeBytes(mDefaultSizeBytes) { - mDataStorePath = db_folder; - std::string db_pathname = makeFullPath(db_filename); - - // simple flags for the moment - these will likely be extended - // later on to support the SQLite mutex model for reading/writing - // simultaneously - perhaps when we look at supporting textures too - int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; - - if (sqlite3_open_v2( - db_pathname.c_str(), - &mDb, - flags, - nullptr // Name of VFS module to use - ) != SQLITE_OK) - { - printError(__FUNCDNAME__ , "open_v2", true); - close(); - return false; - } - - // I elected to store the code that generates the statement - // in sepsrate functions throughout - this seemed like a cleaner - // approach than having hundreds of stmt << "REPLACE AS" lines - // interspersed in the logic code. They are all prefixed with - // 'sqlCompose' and followed by a short description. - const std::string stmt = sqlComposeCreateTable(); - if (! sqliteExec(stmt, __FUNCDNAME__ )) + // no access to LLControlGroup / gSavedSettings so use the system environment + // to trigger additional debugging on top of the default "we purged the cache" + mCacheDebugInfo = true; + if (getenv("LL_CACHE_DEBUGGING") != nullptr) { - // Not sure if we need close here - if open fails, then we - // perhaps don't need to close it - TODO: look in SQLite docs - close(); - return false; + mCacheDebugInfo = true; } - return true; + // create cache dir if it does not exist + boost::filesystem::create_directory(cache_dir); } -/////////////////////////////////////////////////////////////////////////////// -// Determines if an entry exists. -// Pass in the id and a variable that will indicate existance -// Returns true/false if function succeeded and the boolean -// you pass in will be set to true/false if entry exists or not -bool llDiskCache::exists(const std::string id, bool& exists) +LLDiskCache::~LLDiskCache() { - if (!mDb) - { - printError(__FUNCDNAME__ , "mDb is invalid", false); - return false; - } - - if (id.empty()) - { - printError(__FUNCDNAME__ , "id is empty", false); - return false; - } - - // As well as the separate function to compose the SQL statement, - // worth pointing out that the code to 'prepare' and 'step' the - // SQLite "prepared statement" has been factored out into its own - // function and used in several other functions. - const std::string stmt = sqlComposeExists(id); - sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_ROW); - if (! prepared_statement) - { - return false; - } - - int result_column_index = 0; - int result_count = sqlite3_column_int(prepared_statement, result_column_index); - if (sqlite3_finalize(prepared_statement) != SQLITE_OK) - { - printError(__FUNCDNAME__ , "sqlite3_finalize()", true); - return false; - } - - // given the uniqueness of the ID, this can only ever be - // either 1 or 0 so this might be a bit confusing - exists = result_count > 0 ? true : false; - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Given an id (likely a UUID + decoration but can be any string), a -// pointer to a blob of binary data along with its length, write the -// entry to the cache -// Returns true/false for success/failure -bool llDiskCache::put(const std::string id, - char* binary_data, - int binary_data_size) -{ - if (!mDb) - { - printError(__FUNCDNAME__ , "mDb is invalid", false); - return false; - } - - if (id.empty()) - { - printError(__FUNCDNAME__ , "id is empty", false); - return false; - } - - // < 0 is obvious but we assert the data must be at least 1 byte long - if (binary_data_size <= 0) - { - printError(__FUNCDNAME__ , "size of binary file to write is invalid", false); - return false; - } - - // we generate a unique filename for the actual data itself - // which is stored on disk directly and not in the database. - // TODO: consider making the filename more like the ID passed in - // although the problem with that is we would have to parse the id - // to remove invalid filename chars, consider length etc. As it - // stands, we can write a simple SQL statement to return the filename - // given the ID. - const std::string filename = makeUniqueFilename(); - const std::string filepath = makeFullPath(filename); - std::ofstream file(filepath, std::ios::out | std::ios::binary); - if (! file.is_open()) - { - std::ostringstream error; - error << "Unable to open " << filepath << " for writing"; - printError(__FUNCDNAME__ , error.str(), false); - return false; - } - - file.write((char*)binary_data, binary_data_size); - file.close(); - - // I think is a catchall "wasn't able to write the file to disk" - // conditional but should revisit when we hook this up to viewer - // code to make sure - we never want to write bad/no data to the - // disk and not indicate something has gone wrong - if (file.bad()) - { - std::ostringstream error; - error << "Unable to write " << binary_data_size; - error << " bytes to " << filepath; - printError(__FUNCDNAME__ , error.str(), false); - - return false; - } - - // this is where the filename/size is written to the database along - // with the current date/time for the created/last access times - const std::string stmt = sqlComposePut(id, filename, binary_data_size); - if (! sqlitePrepareStep(__FUNCDNAME__ , stmt, SQLITE_DONE)) - { - return false; - } - - return true; } -/////////////////////////////////////////////////////////////////////////////// -// Given an id (likely a UUID + decoration but can be any string), the -// address of a pointer that will be used to allocate memory and a -// varialble that will contain the length of the data, get an item from -// the cache. Note that the memory buffer returned belongs to the calling -// function and it is its responsiblity to clean it up when it's no -// longer needed. -// Returns true/false for success/failure -const bool llDiskCache::get(const std::string id, - char** mem_buffer, - int& mem_buffer_size) +void LLDiskCache::purge() { - // Check if the entry exists first to avoid dealing with a bunch - // of conditions that look like failure but aren't in the main code. - // Note exists() is a public method and also tests for mDB and id - // being valid so we can safely put this about the same tests - // in this function - bool get_exists = false; - if (! exists(id, get_exists)) + if (mCacheDebugInfo) { - return false; - } - if (! get_exists) - { - return false; + LL_INFOS() << "Total dir size before purge is " << dirFileSize(mCacheDir) << LL_ENDL; } - if (!mDb) - { - printError(__FUNCDNAME__ , "mDb is invalid", false); - return false; - } - - if (id.empty()) - { - printError(__FUNCDNAME__ , "id is empty", false); - return false; - } + auto start_time = std::chrono::high_resolution_clock::now(); - const std::string stmt_select = sqlComposeGetSelect(id); - sqlite3_stmt* prepared_statement = sqlitePrepareStep(__FUNCDNAME__ , stmt_select, SQLITE_ROW); - if (! prepared_statement) - { - return false; - } + typedef std::pair> file_info_t; + std::vector file_info; - int result_column_index = 0; - const unsigned char* text = sqlite3_column_text(prepared_statement, result_column_index); - if (text == nullptr) + if (boost::filesystem::is_directory(mCacheDir)) { - printError(__FUNCDNAME__ , "filename is nullptr", true); - return false; - } - const std::string filename = std::string(reinterpret_cast(text)); - const std::string filepath = makeFullPath(filename); + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {})) + { + if (boost::filesystem::is_regular_file(entry)) + { + uintmax_t file_size = boost::filesystem::file_size(entry); + const std::string file_path = entry.path().string(); + const std::time_t file_time = boost::filesystem::last_write_time(entry); - result_column_index = 1; - int filesize_db = sqlite3_column_int(prepared_statement, result_column_index); - if (filesize_db <= 0) - { - printError(__FUNCDNAME__ , "filesize is invalid", true); - return false; + file_info.push_back(file_info_t(file_time, { file_size, file_path })); + } + } } - if (sqlite3_finalize(prepared_statement) != SQLITE_OK) + std::sort(file_info.begin(), file_info.end(), [](file_info_t& x, file_info_t& y) { - printError(__FUNCDNAME__ , "sqlite3_finalize()", true); - return false; - } + return x.first > y.first; + }); - // Now we have the fiename, we can read the file from disk - std::ifstream file(filepath, std::ios::in | std::ios::binary | std::ios::ate); - if (! file.is_open()) - { - std::ostringstream error; - error << "Unable to open " << filepath << " for reading"; - printError(__FUNCDNAME__ , error.str(), false); - return false; - } + LL_INFOS() << "Purging cache to a maximum of " << mMaxSizeBytes << " bytes" << LL_ENDL; - // we get the expected filesize from the database but we can also - // get it (easily) when we read the file from the disk. We compare - // the two and return false if they don't match - std::streampos filesize_file = file.tellg(); - if (filesize_db != filesize_file) + uintmax_t file_size_total = 0; + for (file_info_t& entry : file_info) { - std::ostringstream error; - error << "File size from DB (" << filesize_db << ")"; - error << " and "; - error << "file size from file (" << filesize_file << ")"; - error << " in file " << filepath << " are different"; - printError(__FUNCDNAME__ , error.str(), false); - - return false; - } - - // doest matter if we choose DB or file version - they must be - // identical if we get this far - just used for clarity - int filesize = filesize_db; + file_size_total += entry.second.first; - // allocate a block of memory that we pass back for the calling - // function to use then delete when it's no longer needed - *mem_buffer = new char[filesize]; - mem_buffer_size = filesize; + std::string action = ""; + if (file_size_total > mMaxSizeBytes) + { + action = "DELETE:"; + boost::filesystem::remove(entry.second.second); + } + else + { + action = " KEEP:"; + } - file.seekg(0, std::ios::beg); - file.read(*mem_buffer, filesize); - file.close(); + if (mCacheDebugInfo) + { + // have to do this because of LL_INFO/LL_END weirdness + std::ostringstream line; - if (file.bad()) - { - std::ostringstream error; - error << "Unable to read " << filesize; - error << " bytes from " << filepath; - printError(__FUNCDNAME__ , error.str(), false); - - return false; + line << action << " "; + line << entry.first << " "; + line << entry.second.first << " "; + line << entry.second.second; + line << " (" << file_size_total << "/" << mMaxSizeBytes << ")"; + LL_INFOS() << line.str() << LL_ENDL; + } } - // here we update the count of times the file is accessed so - // we can keep track of how many times it's been requested. - // This will be useful for metrics and perhaps determining - // if a file should not be purged even though its age - // might suggest that it should. - // In addition, this is where the time of last access is updated - // in the database and that us used to determine what is purged - // in an LRU fashion when the purge function is called. - const std::string stmt_update = sqlComposeGetUpdate(id); - if (! sqliteExec(stmt_update, __FUNCDNAME__ )) + if (mCacheDebugInfo) { - return false; + auto end_time = std::chrono::high_resolution_clock::now(); + auto execute_time = std::chrono::duration_cast(end_time - start_time).count(); + LL_INFOS() << "Total dir size after purge is " << dirFileSize(mCacheDir) << LL_ENDL; + LL_INFOS() << "Cache purge took " << execute_time << " ms to execute for " << file_info.size() << " files" << LL_ENDL; } - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Purges the database of older entries using an LRU approach. -// Pass in the number of entries to retain. -// This is called now after open to "clean up" the cache when the -// application starts. -// TODO: IMPORTANT: compose a list of files that will be deleted -// and delete them from disk too - not just from the DB -bool llDiskCache::purge(int num_entries) -{ - if (num_entries < 0) - { - printError(__FUNCDNAME__ , "number of entries to purge is invalid", false); - return false; - } - - // find the rows affected and get the filenames for them -//swww - - // delete oldest entries leaving the correct number in place - const std::string stmt = sqlComposePurge(num_entries); - if (! sqliteExec(stmt, __FUNCDNAME__ )) - { - return false; - } - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Call at application shutdown -void llDiskCache::close() -{ - sqlite3_close(mDb); -} - -/////////////////////////////////////////////////////////////////////////////// -// Determine the version of SQLite in use -// TODO: make this a static so we can get to it from the Viewer About -// box without instantiating the whole thing. -const std::string llDiskCache::dbVersion() -{ - std::ostringstream version; - - version << sqlite3_libversion(); - - return version.str(); } -/////////////////////////////////////////////////////////////////////////////// -// Given an id, return the matching filename -const std::string llDiskCache::getFilenameById(const std::string id) -{ - // TODO: - return std::string(); -} - -/////////////////////////////////////////////////////////////////////////////// -// Given an id, return the number of times that entry has been -// accessed from the cache -const int llDiskCache::getAccessCountById(const std::string id) -{ - // TODO: - return -1; -} - -/////////////////////////////////////////////////////////////////////////////// -// Return the number of entries currently in the cache as well as -// the maximum possible entries. -void llDiskCache::getNumEntries(int& num_entries, int& max_entries) -{ - num_entries = -1; - max_entries = -1; -} - -/////////////////////////////////////////////////////////////////////////////// -// Wraps the sqlite3_exec(..) used in many places -bool llDiskCache::sqliteExec(const std::string stmt, - const std::string funcname) -{ - if (sqlite3_exec( - mDb, - stmt.c_str(), - nullptr, // Callback function (unused) - nullptr, // 1st argument to callback (unused) - nullptr // Error msg written here (unused) - ) != SQLITE_OK) - { - printError(funcname, "sqlite3_exec", true); - return false; - } - - return true; -} - -/////////////////////////////////////////////////////////////////////////////// -// Wraps the sqlite3_prepare_v2 and sqlite3_step calls used in many places -sqlite3_stmt* llDiskCache::sqlitePrepareStep(const std::string funcname, - const std::string stmt, - int sqlite_success_condition) -{ - sqlite3_stmt* prepared_statement; - - if (sqlite3_prepare_v2( - mDb, - stmt.c_str(), - -1, // Maximum length of zSql in bytes. - &prepared_statement, - 0 // OUT: Pointer to unused portion of zSql - ) != SQLITE_OK) - { - printError(funcname, "sqlite3_prepare_v2", true); - return nullptr; - } - - if (sqlite3_step(prepared_statement) != sqlite_success_condition) - { - printError(funcname, "sqlite3_step", true); - sqlite3_finalize(prepared_statement); - return nullptr; - } - return prepared_statement; -} - -/////////////////////////////////////////////////////////////////////////////// -// When an "error" occurss - e.g. database cannot be found, file cannot be -// written, invalid argument passed into a function etc. a message is -// written to stderr that should end up in the viewer log -// TODO: Set the verbosity using the usual Viewer mechanism -void llDiskCache::printError(const std::string funcname, - const std::string desc, - bool is_sqlite_err) -{ - std::ostringstream err_msg; - - err_msg << "llDiskCache error in "; - err_msg << __FUNCDNAME__ << "(...) "; - err_msg << desc; - - if (is_sqlite_err) - { - err_msg << " - "; - err_msg << std::string(sqlite3_errmsg(mDb)); - } - - // TODO: set via viewer verbosity level - const int verbosity = 1; - if (verbosity > 0) - { - std::cerr << err_msg.str() << std::endl; - } -} - -/////////////////////////////////////////////////////////////////////////////// -// wrapper for SQLite code to begin an SQL transaction - not used yet but -// it will be eventually -bool llDiskCache::beginTransaction() +/** + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was + * accessed which is used in the mechanism for purging the cache, is up to date. + */ +void LLDiskCache::updateFileAccessTime(const std::string file_path) { - const std::string stmt("BEGIN TRANSACTION"); - if (! sqliteExec(stmt, __FUNCDNAME__ )) - { - return false; - } - - return true; + const std::time_t file_time = std::time(nullptr); + boost::filesystem::last_write_time(file_path, file_time); } -/////////////////////////////////////////////////////////////////////////////// -// wrapper for SQLite code to end an SQL transaction - not used yet but -// it will be eventually -bool llDiskCache::endTransaction() +/** + * Clear the cache by removing all the files in the cache directory + * individually. It's important to maintain control of which directory + * if passed in and not let the user inadvertently (or maliciously) set + * it to an random location like your project source or OS system directory + */ +void LLDiskCache::clearCache(const std::string cache_dir) { - const std::string stmt("COMMIT"); - if (! sqliteExec(stmt, __FUNCDNAME__ )) + if (boost::filesystem::is_directory(cache_dir)) { - return false; + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_dir), {})) + { + if (boost::filesystem::is_regular_file(entry)) + { + boost::filesystem::remove(entry); + } + } } - - return true; } -/////////////////////////////////////////////////////////////////////////////// -// Build a unique filename that will be used to store the actual file -// on disk (as opposed to the meta data in the database) -// TODO: I think this needs more work once we move it to the viewer -// and espcially to make it cross platform - see 'std::tmpnam' -// depreciation comments in compiler output for example -std::string llDiskCache::makeUniqueFilename() +/** + * Utility function to get the total filesize of all files in a directory. It + * used to test file extensions to only check cache files but that was removed. + * There may be a better way that works directly on the folder (similar to + * right clicking on a folder in the OS and asking for size vs right clicking + * on all files and adding up manually) but this is very fast - less than 100ms + * in my testing so, so long as it's not called frequently, it should be okay. + * Note that's it's only currently used for logging/debugging so if performance + * is ever an issue, optimizing this or removing it altogether, is an easy win. + */ +uintmax_t LLDiskCache::dirFileSize(const std::string dir) { - // TODO: replace with boost - this is marked as deprecated? - std::string base = std::tmpnam(nullptr); - - // C++11 random number generation!!! - static std::random_device dev; - static std::mt19937 rng(dev()); - std::uniform_int_distribution dist(100000, 999999); + uintmax_t total_file_size = 0; - // currently the tmp filename from std::tmpnam() on macOS - // is of the form `/tmp/foo/bar.12345 and the following code - // strips all the preceding dirs - we likely want a more - // robust (and cross platform solution) when we move to the - // viewer code - std::size_t found = base.rfind(systemSeparator()); - if (found != std::string::npos && found < base.size() - 2) - { - base = base.substr(found + 1); - } - else + if (boost::filesystem::is_directory(dir)) { - base = ""; + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir), {})) + { + if (boost::filesystem::is_regular_file(entry)) + { + total_file_size += boost::filesystem::file_size(entry); + } + } } - // we mix in a random number for some more entropy.. - // (i know, i know...) - std::ostringstream unique_filename; - unique_filename << base; - unique_filename << "."; - unique_filename << dist(rng); - - return unique_filename.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// Return system file/path separator - likely replaced by the version -// in the viewer -const std::string llDiskCache::systemSeparator() -{ -// TODO: replace in viewer with relevant call -#ifdef _WIN32 - return "\\"; -#else - return "/"; -#endif -} - -/////////////////////////////////////////////////////////////////////////////// -// Given a filename, compose a full path based on the path name passed -// in when the database was opened and the separator in play. -const std::string llDiskCache::makeFullPath(const std::string filename) -{ - std::string pathname = mDataStorePath + systemSeparator() + filename; - - return pathname; -} - -/////////////////////////////////////////////////////////////////////////////// -// -const std::string llDiskCache::sqlComposeCreateTable() -{ - std::ostringstream stmt; - stmt << "CREATE TABLE IF NOT EXISTS "; - stmt << mTableName; - stmt << "("; - stmt << mIdFieldName; - stmt << " TEXT PRIMARY KEY, "; - stmt << mFilenameFieldName; - stmt << " TEXT, "; - stmt << mFilesizeFieldName; - stmt << " INTEGER DEFAULT 0, "; - stmt << mInsertionDateTimeFieldName; - stmt << " TEXT, "; - stmt << mLastAccessDateTimeFieldName; - stmt << " TEXT, "; - stmt << mAccessCountFieldName; - stmt << " INTEGER DEFAULT 0"; - stmt << ")"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// -const std::string llDiskCache::sqlComposeExists(const std::string id) -{ - std::ostringstream stmt; - stmt << "SELECT COUNT(*) FROM "; - stmt << mTableName; - stmt << " WHERE "; - stmt << mIdFieldName; - stmt << "='"; - stmt << id; - stmt << "'"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement to write an entry to the DB -// Saves id, filename (generated), file size and create/last access date -const std::string llDiskCache::sqlComposePut(const std::string id, - const std::string filename, - int binary_data_size) -{ - std::ostringstream stmt; - stmt << "REPLACE INTO "; - stmt << mTableName; - stmt << "("; - stmt << mIdFieldName; - stmt << ","; - stmt << mFilenameFieldName; - stmt << ","; - stmt << mFilesizeFieldName; - stmt << ","; - stmt << mInsertionDateTimeFieldName; - stmt << ","; - stmt << mLastAccessDateTimeFieldName; - stmt << ") "; - stmt << "VALUES("; - stmt << "'"; - stmt << id; - stmt << "', "; - stmt << "'"; - stmt << filename; - stmt << "', "; - stmt << binary_data_size; - stmt << ", "; - stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; - stmt << ", "; - stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; - stmt << ")"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement used in get() to look up the filename and file size -const std::string llDiskCache::sqlComposeGetSelect(const std::string id) -{ - std::ostringstream stmt; - stmt << "SELECT "; - stmt << mFilenameFieldName; - stmt << ", "; - stmt << mFilesizeFieldName; - stmt << " FROM "; - stmt << mTableName; - stmt << " WHERE "; - stmt << mIdFieldName; - stmt << "='"; - stmt << id; - stmt << "'"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement to update the date/time of last access as well as the -// count of number of times the file has been accessed. -// Note: the more accurate representation of date/time is used to -// ensure ms accuracy vs the standard INTEGER days since epoch approach -const std::string llDiskCache::sqlComposeGetUpdate(const std::string id) -{ - std::ostringstream stmt; - stmt << "UPDATE "; - stmt << mTableName; - stmt << " SET "; - stmt << mAccessCountFieldName; - stmt << "="; - stmt << mAccessCountFieldName; - stmt << "+1"; - stmt << ", "; - stmt << mLastAccessDateTimeFieldName; - stmt << "="; - stmt << "strftime('%Y-%m-%d %H:%M:%f', 'now')"; - stmt << " WHERE "; - stmt << mIdFieldName; - stmt << "='"; - stmt << id; - stmt << "'"; - -#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -#endif - - return stmt.str(); -} - -/////////////////////////////////////////////////////////////////////////////// -// SQL statement to remove items from the database that are older -// than the newest num_elements entries -const std::string llDiskCache::sqlComposePurge(int num_entries) -{ - std::ostringstream stmt; - stmt << "DELETE FROM "; - stmt << mTableName; - stmt << " WHERE "; - stmt << mLastAccessDateTimeFieldName; - stmt << " NOT IN "; - stmt << "("; - stmt << "SELECT "; - stmt << mLastAccessDateTimeFieldName; - stmt << " FROM "; - stmt << mTableName; - stmt << " ORDER BY "; - stmt << mLastAccessDateTimeFieldName; - stmt << " DESC"; - stmt << " LIMIT "; - stmt << num_entries; - stmt << ")"; - -//#ifdef SHOW_STATEMENTS - std::cout << stmt.str() << std::endl; -//#endif - - return stmt.str(); + return total_file_size; } diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 39b8f7ef48..1618cec6a6 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -1,11 +1,6 @@ /** * @file lldiskcache.h - * @brief Declaration SQLite meta data / file storage API - * @brief Declaration of the generic disk cache interface - as well the SQLite header/implementation. - * @Note Eventually, this component might split into an interface - * file and multiple implemtations but for now, this is the - * only one so I think it's okay to combine everything. + * @brief The disk cache implementation. * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code @@ -32,99 +27,68 @@ #ifndef _LLDISKCACHE #define _LLDISKCACHE -#include -#include +class LLDiskCache +{ + public: + LLDiskCache(const std::string cache_dir); + ~LLDiskCache(); -#include "sqlite3.h" + /** + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was + * accessed which is used in the mechanism for purging the cache, is up to date. + */ + void updateFileAccessTime(const std::string file_path); -// Enable this line to display each SQL statement when it is executed -// It can lead to a lot of spam but useful for debugging -//#define SHOW_STATEMENTS + /** + * Purge the oldest items in the cache so that the combined size of all files + * is no bigger than mMaxSizeBytes. + */ + void purge(); -// I toyed with the idea of a variety of approaches and thought have -// an abstract base class with which to hang different implementations -// off was a good idea but I think we settled on a single approach -// so I will likely remove this level of indirection if not other -// interesting implementation ideas present themselves. -class llDiskCacheBase -{ - public: - llDiskCacheBase() {}; - virtual ~llDiskCacheBase() {}; - virtual bool open(const std::string db_folder, - const std::string db_filename) = 0; - virtual bool exists(const std::string id, bool& exists) = 0; - virtual bool put(const std::string id, - char* binary_data, - int binary_data_size) = 0; - virtual const bool get(const std::string id, - char** mem_buffer, - int& mem_buffer_size) = 0; - virtual bool purge(int num_entries) = 0; - virtual void close() = 0; - virtual const std::string dbVersion() = 0; - virtual const std::string getFilenameById(const std::string id) = 0; - virtual const int getAccessCountById(const std::string id) = 0; - virtual void getNumEntries(int& num_entries, int& max_entries) = 0; -}; + /** + * Clear the cache by removing the files in the cache directory individually + */ + void clearCache(const std::string cache_dir); -// implementation for the SQLite/disk blended case -// see source file for detailed comments -class llDiskCache : - public llDiskCacheBase -{ - public: - llDiskCache(); - virtual ~llDiskCache(); - virtual bool open(const std::string db_folder, - const std::string db_filename) override; - virtual bool exists(const std::string id, bool& exists) override; - virtual bool put(const std::string id, - char* binary_data, - int binary_data_size) override; - virtual const bool get(const std::string id, - char** mem_buffer, - int& mem_buffer_size) override; - virtual bool purge(int num_entries) override; - virtual void close() override; - virtual const std::string dbVersion() override; - virtual const std::string getFilenameById(const std::string id) override; - virtual const int getAccessCountById(const std::string id) override; - virtual void getNumEntries(int& num_entries, int& max_entries) override; + /** + * Manage max size in bytes of cache - use a discrete setter/getter so the value can + * be changed in the preferences and cache cleared/purged without restarting viewer + * to instantiate this class again. + */ + void setMaxSizeBytes(const uintmax_t size_bytes) { mMaxSizeBytes = size_bytes; } + uintmax_t getMaxSizeBytes() const { return mMaxSizeBytes; } private: - sqlite3* mDb; - std::string mDataStorePath; - const std::string mTableName = "lldiskcache"; - const std::string mIdFieldName = "id"; - const std::string mFilenameFieldName = "filename"; - const std::string mFilesizeFieldName = "filesize"; - const std::string mInsertionDateTimeFieldName = "insert_datetime"; - const std::string mLastAccessDateTimeFieldName = "last_access_datetime"; - const std::string mAccessCountFieldName = "access_count"; + /** + * Utility function to gather the total size the files in a given + * directory. Primarily used here to determine the directory size + * before and after the cache purge + */ + uintmax_t dirFileSize(const std::string dir); private: - void printError(const std::string funcname, - const std::string desc, - bool is_sqlite_err); - bool beginTransaction(); - bool endTransaction(); - std::string makeUniqueFilename(); - const std::string systemSeparator(); - const std::string makeFullPath(const std::string filename); - bool sqliteExec(const std::string stmt, - const std::string funcname); - sqlite3_stmt* sqlitePrepareStep(const std::string funcname, - const std::string stmt, - int sqlite_success_condition); - const std::string sqlComposeCreateTable(); - const std::string sqlComposeExists(const std::string id); - const std::string sqlComposePut(const std::string id, - const std::string filename, - int binary_data_size); - const std::string sqlComposeGetSelect(const std::string id); - const std::string sqlComposeGetUpdate(const std::string id); - const std::string sqlComposePurge(int num_entries); + /** + * Default of 20MB seems reasonable - it will likely be reset + * immediately anyway using a value from the Viewer settings + */ + const uintmax_t mDefaultSizeBytes = 20 * 1024 * 1024; + uintmax_t mMaxSizeBytes; + + /** + * The folder that holds the cached files. The consumer of this + * class must avoid letting the user set this location as a malicious + * setting could potentially point it at a non-cache directory (for example, + * the Windows System dir) with disastrous results. + */ + std::string mCacheDir; + + /** + * This is set from the CacheDebugInfo global setting and + * when enabled, displays additional debugging information in + * various parts of the code + */ + bool mCacheDebugInfo; }; #endif // _LLDISKCACHE diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index f0037c9a22..ffc3dee12b 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -42,6 +42,8 @@ const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFile const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); +LLDiskCache* LLFileSystem::mDiskCache = 0; +std::string LLFileSystem::mCacheDirName = "cache"; LLFileSystem::LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) { @@ -104,7 +106,7 @@ const std::string assetTypeToString(LLAssetType::EType at) return std::string("UNKNOWN"); } -const std::string idToFilepath(const std::string id, LLAssetType::EType at) +const std::string LLFileSystem::idToFilepath(const std::string id, LLAssetType::EType at) { /** * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of @@ -117,7 +119,7 @@ const std::string idToFilepath(const std::string id, LLAssetType::EType at) ss << assetTypeToString(at); ss << ".txt"; - const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, ss.str()); + const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName, ss.str()); return filepath; } @@ -289,7 +291,6 @@ BOOL LLFileSystem::write(const U8 *buffer, S32 bytes) BOOL LLFileSystem::writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type) { LLFileSystem file(uuid, type, LLFileSystem::WRITE); - file.setMaxSize(bytes); return file.write(buffer, bytes); } @@ -339,13 +340,6 @@ S32 LLFileSystem::getMaxSize() return INT_MAX; } -BOOL LLFileSystem::setMaxSize(S32 size) -{ - // we don't care what the max size is so we do nothing - // and return true to indicate all was okay - return TRUE; -} - BOOL LLFileSystem::rename(const LLUUID &new_id, const LLAssetType::EType new_type) { LLFileSystem::renameFile(mFileID, mFileType, new_id, new_type); @@ -366,21 +360,15 @@ BOOL LLFileSystem::remove() // static void LLFileSystem::initClass() { -} + const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName); -// static -void LLFileSystem::cleanupClass() -{ -} + LLFileSystem::mDiskCache = new LLDiskCache(cache_dir); -bool LLFileSystem::isLocked() -{ - // I don't think we care about this test since there is no locking - // TODO: remove this function and calling sites? - return FALSE; + mDiskCache->purge(); } -void LLFileSystem::waitForLock() +// static +void LLFileSystem::cleanupClass() { - // TODO: remove this function and calling sites? + delete LLFileSystem::mDiskCache; } diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h index 9cddef74a7..789f9456c7 100644 --- a/indra/llfilesystem/llfilesystem.h +++ b/indra/llfilesystem/llfilesystem.h @@ -32,6 +32,7 @@ #include "lluuid.h" #include "llassettype.h" +#include "lldiskcache.h" class LLFileSystem { @@ -51,13 +52,9 @@ public: S32 getSize(); S32 getMaxSize(); - BOOL setMaxSize(S32 size); BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); BOOL remove(); - bool isLocked(); - void waitForLock(); - static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, @@ -80,6 +77,11 @@ protected: S32 mPosition; S32 mMode; S32 mBytesRead; + +private: + static const std::string idToFilepath(const std::string id, LLAssetType::EType at); + static std::string mCacheDirName; + static LLDiskCache* mDiskCache; }; #endif // LL_FILESYSTEM_H diff --git a/indra/llmessage/lltransfertargetvfile.cpp b/indra/llmessage/lltransfertargetvfile.cpp index 471d687d67..f6faadf87f 100644 --- a/indra/llmessage/lltransfertargetvfile.cpp +++ b/indra/llmessage/lltransfertargetvfile.cpp @@ -141,7 +141,6 @@ LLTSCode LLTransferTargetVFile::dataCallback(const S32 packet_id, U8 *in_datap, LLFileSystem vf(mTempID, mParams.getAssetType(), LLFileSystem::APPEND); if (mNeedsCreate) { - vf.setMaxSize(mSize); mNeedsCreate = FALSE; } diff --git a/indra/llmessage/llxfer_vfile.cpp b/indra/llmessage/llxfer_vfile.cpp index 9de9ed379b..12419b342d 100644 --- a/indra/llmessage/llxfer_vfile.cpp +++ b/indra/llmessage/llxfer_vfile.cpp @@ -261,7 +261,6 @@ void LLXfer_VFile::setXferSize (S32 xfer_size) if (! mVFile) { LLFileSystem file(mTempID, mType, LLFileSystem::APPEND); - file.setMaxSize(xfer_size); } } diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index c45d6dd7af..0123bc32af 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -1351,6 +1351,17 @@ Value 23 + CacheDebugInfo + + Comment + When enabled, display additional cache debugging information + Persist + 1 + Type + Boolean + Value + 0 + CacheLocation Comment diff --git a/indra/newview/llfloaterbvhpreview.cpp b/indra/newview/llfloaterbvhpreview.cpp index 08f3b577b4..687d820a18 100644 --- a/indra/newview/llfloaterbvhpreview.cpp +++ b/indra/newview/llfloaterbvhpreview.cpp @@ -1000,7 +1000,6 @@ void LLFloaterBvhPreview::onBtnOK(void* userdata) LLFileSystem file(motionp->getID(), LLAssetType::AT_ANIMATION, LLFileSystem::APPEND); S32 size = dp.getCurrentSize(); - file.setMaxSize(size); if (file.write((U8*)buffer, size)) { std::string name = floaterp->getChild("name_form")->getValue().asString(); diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index a0d591dc47..3183e6d8fd 100644 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -3241,7 +3241,7 @@ void LLMeshHeaderHandler::processData(LLCore::BufferArray * /* body */, S32 /* b data_size = llmin(data_size, bytes); LLFileSystem file(mesh_id, LLAssetType::AT_MESH, LLFileSystem::WRITE); - if (file.getMaxSize() >= bytes || file.setMaxSize(bytes)) + if (file.getMaxSize() >= bytes) { LLMeshRepository::sCacheBytesWritten += data_size; ++LLMeshRepository::sCacheWrites; diff --git a/indra/newview/llpreviewgesture.cpp b/indra/newview/llpreviewgesture.cpp index 371153aac3..4318a55704 100644 --- a/indra/newview/llpreviewgesture.cpp +++ b/indra/newview/llpreviewgesture.cpp @@ -1140,7 +1140,6 @@ void LLPreviewGesture::saveIfNeeded() LLFileSystem file(assetId, LLAssetType::AT_GESTURE, LLFileSystem::APPEND); S32 size = dp.getCurrentSize(); - file.setMaxSize(size); file.write((U8*)buffer, size); LLLineEditor* descEditor = getChild("desc"); diff --git a/indra/newview/llpreviewnotecard.cpp b/indra/newview/llpreviewnotecard.cpp index 0bccf1d06f..32e1a4a186 100644 --- a/indra/newview/llpreviewnotecard.cpp +++ b/indra/newview/llpreviewnotecard.cpp @@ -563,7 +563,6 @@ bool LLPreviewNotecard::saveIfNeeded(LLInventoryItem* copyitem, bool sync) tid, copyitem); S32 size = buffer.length() + 1; - file.setMaxSize(size); file.write((U8*)buffer.c_str(), size); gAssetStorage->storeAssetData(tid, LLAssetType::AT_NOTECARD, diff --git a/indra/newview/llviewerassetstorage.cpp b/indra/newview/llviewerassetstorage.cpp index df3ff1a3c7..5b76d57196 100644 --- a/indra/newview/llviewerassetstorage.cpp +++ b/indra/newview/llviewerassetstorage.cpp @@ -293,8 +293,6 @@ void LLViewerAssetStorage::storeAssetData( LLFileSystem file(asset_id, asset_type, LLFileSystem::WRITE); - file.setMaxSize(size); - const S32 buf_size = 65536; U8 copy_buf[buf_size]; while ((size = (S32)fread(copy_buf, 1, buf_size, fp))) @@ -528,7 +526,6 @@ void LLViewerAssetStorage::assetRequestCoro( LLUUID temp_id; temp_id.generate(); LLFileSystem vf(temp_id, atype, LLFileSystem::WRITE); - vf.setMaxSize(size); req->mBytesFetched = size; if (!vf.write(raw.data(),size)) { diff --git a/indra/newview/llviewerassetupload.cpp b/indra/newview/llviewerassetupload.cpp index fb3ca69d5d..67ee06e255 100644 --- a/indra/newview/llviewerassetupload.cpp +++ b/indra/newview/llviewerassetupload.cpp @@ -474,8 +474,6 @@ LLSD LLNewFileResourceUploadInfo::exportTempFile() { LLFileSystem file(getAssetId(), assetType, LLFileSystem::WRITE); - file.setMaxSize(file_size); - const S32 buf_size = 65536; U8 copy_buf[buf_size]; while ((file_size = infile.read(copy_buf, buf_size))) @@ -568,7 +566,6 @@ LLSD LLBufferedAssetUploadInfo::prepareUpload() LLFileSystem file(getAssetId(), getAssetType(), LLFileSystem::APPEND); S32 size = mContents.length() + 1; - file.setMaxSize(size); file.write((U8*)mContents.c_str(), size); mStoredToCache = true; -- cgit v1.3 From 08dfc0836fb12855d0c07d811e2909400d5b74f3 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 7 Oct 2020 15:25:12 -0700 Subject: This changeset hooks up many things that have been in progress and moves things about between llfilesystem and lldiskcache - there is still some bookkeeping work left but this is the first version that appears to work and actively manage the cache --- indra/llfilesystem/lldiskcache.cpp | 134 +++++++++++++----- indra/llfilesystem/lldiskcache.h | 82 ++++++++--- indra/llfilesystem/llfilesystem.cpp | 238 ++++++++++---------------------- indra/llfilesystem/llfilesystem.h | 82 +++++------ indra/newview/app_settings/settings.xml | 26 +++- indra/newview/llappviewer.cpp | 15 +- 6 files changed, 307 insertions(+), 270 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 4b2ba0dd79..4b82cf3cce 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -25,41 +25,31 @@ */ #include "linden_common.h" -#include "lluuid.h" +#include "llassettype.h" #include "lldir.h" - -#include "lldiskcache.h" - #include #include - #include -#include -#include -LLDiskCache::LLDiskCache(const std::string cache_dir) : +#include "lldiskcache.h" + +LLDiskCache::LLDiskCache(const std::string cache_dir, + const int max_size_bytes, + const bool enable_cache_debug_info) : mCacheDir(cache_dir), - mMaxSizeBytes(mDefaultSizeBytes) + mMaxSizeBytes(max_size_bytes), + mEnableCacheDebugInfo(enable_cache_debug_info) { - // no access to LLControlGroup / gSavedSettings so use the system environment - // to trigger additional debugging on top of the default "we purged the cache" - mCacheDebugInfo = true; - if (getenv("LL_CACHE_DEBUGGING") != nullptr) - { - mCacheDebugInfo = true; - } + // the prefix used for cache filenames to disambiguate them from other files + mCacheFilenamePrefix = "sl_cache"; // create cache dir if it does not exist boost::filesystem::create_directory(cache_dir); } -LLDiskCache::~LLDiskCache() -{ -} - void LLDiskCache::purge() { - if (mCacheDebugInfo) + if (mEnableCacheDebugInfo) { LL_INFOS() << "Total dir size before purge is " << dirFileSize(mCacheDir) << LL_ENDL; } @@ -75,11 +65,14 @@ void LLDiskCache::purge() { if (boost::filesystem::is_regular_file(entry)) { - uintmax_t file_size = boost::filesystem::file_size(entry); - const std::string file_path = entry.path().string(); - const std::time_t file_time = boost::filesystem::last_write_time(entry); + if (entry.path().string().find(mCacheFilenamePrefix) != std::string::npos) + { + uintmax_t file_size = boost::filesystem::file_size(entry); + const std::string file_path = entry.path().string(); + const std::time_t file_time = boost::filesystem::last_write_time(entry); - file_info.push_back(file_info_t(file_time, { file_size, file_path })); + file_info.push_back(file_info_t(file_time, { file_size, file_path })); + } } } } @@ -107,7 +100,7 @@ void LLDiskCache::purge() action = " KEEP:"; } - if (mCacheDebugInfo) + if (mEnableCacheDebugInfo) { // have to do this because of LL_INFO/LL_END weirdness std::ostringstream line; @@ -121,7 +114,7 @@ void LLDiskCache::purge() } } - if (mCacheDebugInfo) + if (mEnableCacheDebugInfo) { auto end_time = std::chrono::high_resolution_clock::now(); auto execute_time = std::chrono::duration_cast(end_time - start_time).count(); @@ -130,9 +123,78 @@ void LLDiskCache::purge() } } +const std::string LLDiskCache::assetTypeToString(LLAssetType::EType at) +{ + /** + * Make use of the C++17 (or is it 14) feature that allows + * for inline initialization of an std::map<> + */ + typedef std::map asset_type_to_name_t; + asset_type_to_name_t asset_type_to_name = + { + { LLAssetType::AT_TEXTURE, "TEXTURE" }, + { LLAssetType::AT_SOUND, "SOUND" }, + { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, + { LLAssetType::AT_LANDMARK, "LANDMARK" }, + { LLAssetType::AT_SCRIPT, "SCRIPT" }, + { LLAssetType::AT_CLOTHING, "CLOTHING" }, + { LLAssetType::AT_OBJECT, "OBJECT" }, + { LLAssetType::AT_NOTECARD, "NOTECARD" }, + { LLAssetType::AT_CATEGORY, "CATEGORY" }, + { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, + { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, + { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, + { LLAssetType::AT_BODYPART, "BODYPART" }, + { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, + { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, + { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, + { LLAssetType::AT_ANIMATION, "ANIMATION" }, + { LLAssetType::AT_GESTURE, "GESTURE" }, + { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, + { LLAssetType::AT_LINK, "LINK" }, + { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, + { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, + { LLAssetType::AT_WIDGET, "WIDGET" }, + { LLAssetType::AT_PERSON, "PERSON" }, + { LLAssetType::AT_MESH, "MESH" }, + { LLAssetType::AT_SETTINGS, "SETTINGS" }, + { LLAssetType::AT_UNKNOWN, "UNKNOWN" } + }; + + asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); + if (iter != asset_type_to_name.end()) + { + return iter->second; + } + + return std::string("UNKNOWN"); +} + +const std::string LLDiskCache::metaDataToFilepath(const std::string id, + LLAssetType::EType at, + const std::string extra_info) +{ + std::ostringstream file_path; + + file_path << mCacheDir; + file_path << gDirUtilp->getDirDelimiter(); + file_path << mCacheFilenamePrefix; + file_path << "_"; + file_path << id; + file_path << "_"; + file_path << (extra_info.empty() ? "0" : extra_info); + file_path << "_"; + file_path << assetTypeToString(at); + file_path << ".asset"; + + LL_INFOS() << "filepath.str() = " << file_path.str() << LL_ENDL; + + return file_path.str(); +} + /** - * Update the "last write time" of a file to "now". This must be called whenever a - * file in the cache is read (not written) so that the last time the file was + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was * accessed which is used in the mechanism for purging the cache, is up to date. */ void LLDiskCache::updateFileAccessTime(const std::string file_path) @@ -144,8 +206,8 @@ void LLDiskCache::updateFileAccessTime(const std::string file_path) /** * Clear the cache by removing all the files in the cache directory * individually. It's important to maintain control of which directory - * if passed in and not let the user inadvertently (or maliciously) set - * it to an random location like your project source or OS system directory + * if passed in and not let the user inadvertently (or maliciously) set + * it to an random location like your project source or OS system directory */ void LLDiskCache::clearCache(const std::string cache_dir) { @@ -155,7 +217,10 @@ void LLDiskCache::clearCache(const std::string cache_dir) { if (boost::filesystem::is_regular_file(entry)) { - boost::filesystem::remove(entry); + if (entry.path().string().find(mCacheFilenamePrefix) != std::string::npos) + { + boost::filesystem::remove(entry); + } } } } @@ -181,7 +246,10 @@ uintmax_t LLDiskCache::dirFileSize(const std::string dir) { if (boost::filesystem::is_regular_file(entry)) { - total_file_size += boost::filesystem::file_size(entry); + if (entry.path().string().find(mCacheFilenamePrefix) != std::string::npos) + { + total_file_size += boost::filesystem::file_size(entry); + } } } } diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 1618cec6a6..12599a132e 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -27,15 +27,50 @@ #ifndef _LLDISKCACHE #define _LLDISKCACHE -class LLDiskCache +#include "llsingleton.h" + +class LLDiskCache : + public LLParamSingleton { public: - LLDiskCache(const std::string cache_dir); - ~LLDiskCache(); + LLSINGLETON(LLDiskCache, + const std::string cache_dir, + const int max_size_bytes, + const bool enable_cache_debug_info); + virtual ~LLDiskCache() = default; + + public: + ///** + // * The location of the cache dir is set in llappviewer at startup via the + // * saved settings parameters. We do not have access to those saved settings + // * here in LLCommon so we must provide an accessor for other classes to use + // * when they need it - e.g. LLFilesystem needs the path so it can write files + // * to it. + // */ + //const std::string getCacheDirName() { return mCacheDir; } + + ///** + // * Each cache filename has a prefix inserted (see definition of the + // * mCacheFilenamePrefix variable below for why) but the LLFileSystem + // * component needs access to it to in order to create the file so we + // * expose it by a getter here. + // */ + //const std::string getCacheFilenamePrefix() { return mCacheFilenamePrefix; } /** - * Update the "last write time" of a file to "now". This must be called whenever a - * file in the cache is read (not written) so that the last time the file was + * Construct a filename and path to it based on the file meta data + * (id, asset type, additional 'extra' info like discard level perhaps) + * Worth pointing out that this function used to be in LLFileSystem but + * so many things had to be pushed back there to accomodate it, that I + * decided to move it here. Still not sure that's completely right. + */ + const std::string metaDataToFilepath(const std::string id, + LLAssetType::EType at, + const std::string extra_info); + + /** + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was * accessed which is used in the mechanism for purging the cache, is up to date. */ void updateFileAccessTime(const std::string file_path); @@ -51,28 +86,26 @@ class LLDiskCache */ void clearCache(const std::string cache_dir); - /** - * Manage max size in bytes of cache - use a discrete setter/getter so the value can - * be changed in the preferences and cache cleared/purged without restarting viewer - * to instantiate this class again. - */ - void setMaxSizeBytes(const uintmax_t size_bytes) { mMaxSizeBytes = size_bytes; } - uintmax_t getMaxSizeBytes() const { return mMaxSizeBytes; } - private: /** * Utility function to gather the total size the files in a given - * directory. Primarily used here to determine the directory size + * directory. Primarily used here to determine the directory size * before and after the cache purge */ uintmax_t dirFileSize(const std::string dir); + /** + * Utility function to convert an LLAssetType enum into a + * string that we use as part of the cache file filename + */ + const std::string assetTypeToString(LLAssetType::EType at); + private: /** - * Default of 20MB seems reasonable - it will likely be reset - * immediately anyway using a value from the Viewer settings + * The maximum size of the cache in bytes. After purge is called, the + * total size of the cache files in the cache directory will be + * less than this value */ - const uintmax_t mDefaultSizeBytes = 20 * 1024 * 1024; uintmax_t mMaxSizeBytes; /** @@ -84,11 +117,20 @@ class LLDiskCache std::string mCacheDir; /** - * This is set from the CacheDebugInfo global setting and - * when enabled, displays additional debugging information in + * The prefix inserted at the start of a cache file filename to + * help identify it as a cache file. It's probably not required + * (just the presence in the cache folder is enough) but I am + * paranoid about the cache folder being set to something bad + * like the users' OS system dir by mistake or maliciously and + * this will help to offset any damage if that happens. + */ + std::string mCacheFilenamePrefix; + + /** + * When enabled, displays additional debugging information in * various parts of the code */ - bool mCacheDebugInfo; + bool mEnableCacheDebugInfo; }; #endif // _LLDISKCACHE diff --git a/indra/llfilesystem/llfilesystem.cpp b/indra/llfilesystem/llfilesystem.cpp index 6d6ff3d7e1..c6b20caa69 100644 --- a/indra/llfilesystem/llfilesystem.cpp +++ b/indra/llfilesystem/llfilesystem.cpp @@ -1,4 +1,4 @@ -/** +/** * @file filesystem.h * @brief Simulate local file system operations. * @Note The initial implementation does actually use standard C++ @@ -8,21 +8,21 @@ * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2010, Linden Research, Inc. - * + * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License only. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * + * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ */ @@ -36,139 +36,74 @@ #include -const S32 LLFileSystem::READ = 0x00000001; -const S32 LLFileSystem::WRITE = 0x00000002; -const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFileSystem::WRITE -const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE +const S32 LLFileSystem::READ = 0x00000001; +const S32 LLFileSystem::WRITE = 0x00000002; +const S32 LLFileSystem::READ_WRITE = 0x00000003; // LLFileSystem::READ & LLFileSystem::WRITE +const S32 LLFileSystem::APPEND = 0x00000006; // 0x00000004 & LLFileSystem::WRITE static LLTrace::BlockTimerStatHandle FTM_VFILE_WAIT("VFile Wait"); -LLDiskCache* LLFileSystem::mDiskCache = 0; -std::string LLFileSystem::mCacheDirName = "cache"; -LLFileSystem::LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode) +LLFileSystem::LLFileSystem(const LLUUID& file_id, const LLAssetType::EType file_type, S32 mode) { - mFileType = file_type; - mFileID = file_id; - mPosition = 0; + mFileType = file_type; + mFileID = file_id; + mPosition = 0; mBytesRead = 0; - mReadComplete = FALSE; - mMode = mode; + mMode = mode; } LLFileSystem::~LLFileSystem() { } -const std::string assetTypeToString(LLAssetType::EType at) +// static +bool LLFileSystem::getExists(const LLUUID& file_id, const LLAssetType::EType file_type) { - /** - * Make use of the C++17 (or is it 14) feature that allows - * for inline initialization of an std::map<> - */ - typedef std::map asset_type_to_name_t; - asset_type_to_name_t asset_type_to_name = - { - { LLAssetType::AT_TEXTURE, "TEXTURE" }, - { LLAssetType::AT_SOUND, "SOUND" }, - { LLAssetType::AT_CALLINGCARD, "CALLINGCARD" }, - { LLAssetType::AT_LANDMARK, "LANDMARK" }, - { LLAssetType::AT_SCRIPT, "SCRIPT" }, - { LLAssetType::AT_CLOTHING, "CLOTHING" }, - { LLAssetType::AT_OBJECT, "OBJECT" }, - { LLAssetType::AT_NOTECARD, "NOTECARD" }, - { LLAssetType::AT_CATEGORY, "CATEGORY" }, - { LLAssetType::AT_LSL_TEXT, "LSL_TEXT" }, - { LLAssetType::AT_LSL_BYTECODE, "LSL_BYTECODE" }, - { LLAssetType::AT_TEXTURE_TGA, "TEXTURE_TGA" }, - { LLAssetType::AT_BODYPART, "BODYPART" }, - { LLAssetType::AT_SOUND_WAV, "SOUND_WAV" }, - { LLAssetType::AT_IMAGE_TGA, "IMAGE_TGA" }, - { LLAssetType::AT_IMAGE_JPEG, "IMAGE_JPEG" }, - { LLAssetType::AT_ANIMATION, "ANIMATION" }, - { LLAssetType::AT_GESTURE, "GESTURE" }, - { LLAssetType::AT_SIMSTATE, "SIMSTATE" }, - { LLAssetType::AT_LINK, "LINK" }, - { LLAssetType::AT_LINK_FOLDER, "LINK_FOLDER" }, - { LLAssetType::AT_MARKETPLACE_FOLDER, "MARKETPLACE_FOLDER" }, - { LLAssetType::AT_WIDGET, "WIDGET" }, - { LLAssetType::AT_PERSON, "PERSON" }, - { LLAssetType::AT_MESH, "MESH" }, - { LLAssetType::AT_SETTINGS, "SETTINGS" }, - { LLAssetType::AT_UNKNOWN, "UNKNOWN" } - }; - - asset_type_to_name_t::iterator iter = asset_type_to_name.find(at); - if (iter != asset_type_to_name.end()) + std::string id_str; + file_id.toString(id_str); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); + + std::ifstream file(filename, std::ios::binary); + if (file.is_open()) { - return iter->second; + file.seekg(0, std::ios::end); + return file.tellg() > 0; } - - return std::string("UNKNOWN"); -} - -const std::string LLFileSystem::idToFilepath(const std::string id, LLAssetType::EType at) -{ - /** - * For the moment this is just {UUID}_{ASSET_TYPE}.txt but of - * course, will be greatly expanded upon - */ - std::ostringstream ss; - ss << "00cache_"; - ss << id; - ss << "_"; - ss << assetTypeToString(at); - ss << ".txt"; - - const std::string filepath = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName, ss.str()); - - return filepath; + return false; } // static -bool LLFileSystem::getExists(const LLUUID &file_id, const LLAssetType::EType file_type) -{ - std::string id_str; - file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - - std::ifstream file(filename, std::ios::binary); - if (file.is_open()) - { - file.seekg(0, std::ios::end); - return file.tellg() > 0; - } - return false; -} - -// static -bool LLFileSystem::removeFile(const LLUUID &file_id, const LLAssetType::EType file_type) +bool LLFileSystem::removeFile(const LLUUID& file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); - + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); + std::remove(filename.c_str()); return true; } // static -bool LLFileSystem::renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, - const LLUUID &new_file_id, const LLAssetType::EType new_file_type) +bool LLFileSystem::renameFile(const LLUUID& old_file_id, const LLAssetType::EType old_file_type, + const LLUUID& new_file_id, const LLAssetType::EType new_file_type) { std::string old_id_str; old_file_id.toString(old_id_str); - const std::string old_filename = idToFilepath(old_id_str, old_file_type); + const std::string extra_info = ""; + const std::string old_filename = LLDiskCache::getInstance()->metaDataToFilepath(old_id_str, old_file_type, extra_info); std::string new_id_str; new_file_id.toString(new_id_str); - const std::string new_filename = idToFilepath(new_id_str, new_file_type); + const std::string new_filename = LLDiskCache::getInstance()->metaDataToFilepath(new_id_str, new_file_type, extra_info); if (std::rename(old_filename.c_str(), new_filename.c_str())) { // We would like to return FALSE here indicating the operation // failed but the original code does not and doing so seems to - // break a lot of things so we go with the flow... + // break a lot of things so we go with the flow... //return FALSE; } @@ -176,11 +111,12 @@ bool LLFileSystem::renameFile(const LLUUID &old_file_id, const LLAssetType::ETyp } // static -S32 LLFileSystem::getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type) +S32 LLFileSystem::getFileSize(const LLUUID& file_id, const LLAssetType::EType file_type) { std::string id_str; file_id.toString(id_str); - const std::string filename = idToFilepath(id_str, file_type); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, file_type, extra_info); S32 file_size = 0; std::ifstream file(filename, std::ios::binary); @@ -193,15 +129,14 @@ S32 LLFileSystem::getFileSize(const LLUUID &file_id, const LLAssetType::EType fi return file_size; } -BOOL LLFileSystem::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) +BOOL LLFileSystem::read(U8* buffer, S32 bytes) { - BOOL success = TRUE; - - mReadComplete = FALSE; + BOOL success = TRUE; std::string id; mFileID.toString(id); - const std::string filename = idToFilepath(id, mFileType); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id, mFileType, extra_info); std::ifstream file(filename, std::ios::binary); if (file.is_open()) @@ -226,44 +161,33 @@ BOOL LLFileSystem::read(U8 *buffer, S32 bytes, BOOL async, F32 priority) { success = FALSE; } - - mReadComplete = TRUE; } - // update the last access time for the file - this is required + // update the last access time for the file - this is required // even though we are reading and not writing because this is the // way the cache works - it relies on a valid "last accessed time" for // each file so it knows how to remove the oldest, unused files - LLFileSystem::mDiskCache->updateFileAccessTime(filename); + LLDiskCache::getInstance()->updateFileAccessTime(filename); return success; } -BOOL LLFileSystem::isReadComplete() -{ - if (mReadComplete) - { - return TRUE; - } - - return FALSE; -} - S32 LLFileSystem::getLastBytesRead() { - return mBytesRead; + return mBytesRead; } BOOL LLFileSystem::eof() { - return mPosition >= getSize(); + return mPosition >= getSize(); } -BOOL LLFileSystem::write(const U8 *buffer, S32 bytes) +BOOL LLFileSystem::write(const U8* buffer, S32 bytes) { std::string id_str; mFileID.toString(id_str); - const std::string filename = idToFilepath(id_str, mFileType); + const std::string extra_info = ""; + const std::string filename = LLDiskCache::getInstance()->metaDataToFilepath(id_str, mFileType, extra_info); BOOL success = FALSE; @@ -295,37 +219,37 @@ BOOL LLFileSystem::write(const U8 *buffer, S32 bytes) BOOL LLFileSystem::seek(S32 offset, S32 origin) { - if (-1 == origin) - { - origin = mPosition; - } + if (-1 == origin) + { + origin = mPosition; + } - S32 new_pos = origin + offset; + S32 new_pos = origin + offset; - S32 size = getSize(); + S32 size = getSize(); - if (new_pos > size) - { - LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; + if (new_pos > size) + { + LL_WARNS() << "Attempt to seek past end of file" << LL_ENDL; - mPosition = size; - return FALSE; - } - else if (new_pos < 0) - { - LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; + mPosition = size; + return FALSE; + } + else if (new_pos < 0) + { + LL_WARNS() << "Attempt to seek past beginning of file" << LL_ENDL; - mPosition = 0; - return FALSE; - } + mPosition = 0; + return FALSE; + } - mPosition = new_pos; - return TRUE; + mPosition = new_pos; + return TRUE; } S32 LLFileSystem::tell() const { - return mPosition; + return mPosition; } S32 LLFileSystem::getSize() @@ -335,11 +259,11 @@ S32 LLFileSystem::getSize() S32 LLFileSystem::getMaxSize() { - // offer up a huge size since we don't care what the max is + // offer up a huge size since we don't care what the max is return INT_MAX; } -BOOL LLFileSystem::rename(const LLUUID &new_id, const LLAssetType::EType new_type) +BOOL LLFileSystem::rename(const LLUUID& new_id, const LLAssetType::EType new_type) { LLFileSystem::renameFile(mFileID, mFileType, new_id, new_type); @@ -355,19 +279,3 @@ BOOL LLFileSystem::remove() return TRUE; } - -// static -void LLFileSystem::initClass() -{ - const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, mCacheDirName); - - LLFileSystem::mDiskCache = new LLDiskCache(cache_dir); - - mDiskCache->purge(); -} - -// static -void LLFileSystem::cleanupClass() -{ - delete LLFileSystem::mDiskCache; -} diff --git a/indra/llfilesystem/llfilesystem.h b/indra/llfilesystem/llfilesystem.h index 5d87de9bf8..46bf1bd775 100644 --- a/indra/llfilesystem/llfilesystem.h +++ b/indra/llfilesystem/llfilesystem.h @@ -1,5 +1,5 @@ -/** -/** +/** +/** * @file filesystem.h * @brief Simulate local file system operations. * @Note The initial implementation does actually use standard C++ @@ -9,21 +9,21 @@ * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2010, Linden Research, Inc. - * + * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; * version 2.1 of the License only. - * + * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. - * + * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * + * * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA * $/LicenseInfo$ */ @@ -37,51 +37,43 @@ class LLFileSystem { -public: - LLFileSystem(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLFileSystem::READ); - ~LLFileSystem(); + public: + LLFileSystem(const LLUUID& file_id, const LLAssetType::EType file_type, S32 mode = LLFileSystem::READ); + ~LLFileSystem(); - BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ - BOOL isReadComplete(); - S32 getLastBytesRead(); - BOOL eof(); + BOOL read(U8* buffer, S32 bytes); + S32 getLastBytesRead(); + BOOL eof(); - BOOL write(const U8 *buffer, S32 bytes); - BOOL seek(S32 offset, S32 origin = -1); - S32 tell() const; + BOOL write(const U8* buffer, S32 bytes); + BOOL seek(S32 offset, S32 origin = -1); + S32 tell() const; - S32 getSize(); - S32 getMaxSize(); - BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); - BOOL remove(); + S32 getSize(); + S32 getMaxSize(); + BOOL rename(const LLUUID& new_id, const LLAssetType::EType new_type); + BOOL remove(); - static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); - static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); - static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, - const LLUUID &new_file_id, const LLAssetType::EType new_file_type); - static S32 getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type); - - static void initClass(); - static void cleanupClass(); + static bool getExists(const LLUUID& file_id, const LLAssetType::EType file_type); + static bool removeFile(const LLUUID& file_id, const LLAssetType::EType file_type); + static bool renameFile(const LLUUID& old_file_id, const LLAssetType::EType old_file_type, + const LLUUID& new_file_id, const LLAssetType::EType new_file_type); + static S32 getFileSize(const LLUUID& file_id, const LLAssetType::EType file_type); -public: - static const S32 READ; - static const S32 WRITE; - static const S32 READ_WRITE; - static const S32 APPEND; - -protected: - LLAssetType::EType mFileType; - BOOL mReadComplete; - LLUUID mFileID; - S32 mPosition; - S32 mMode; - S32 mBytesRead; + public: + static const S32 READ; + static const S32 WRITE; + static const S32 READ_WRITE; + static const S32 APPEND; -private: - static const std::string idToFilepath(const std::string id, LLAssetType::EType at); - static std::string mCacheDirName; - static LLDiskCache* mDiskCache; + protected: + LLAssetType::EType mFileType; + LLUUID mFileID; + S32 mPosition; + S32 mMode; + S32 mBytesRead; +//private: +// static const std::string idToFilepath(const std::string id, LLAssetType::EType at); }; #endif // LL_FILESYSTEM_H diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 0123bc32af..2e65aef9a2 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -1351,10 +1351,10 @@ Value 23 - CacheDebugInfo + EnableCacheDebugInfo Comment - When enabled, display additional cache debugging information + When set, display additional cache debugging information Persist 1 Type @@ -1362,6 +1362,28 @@ Value 0 + DiskCacheMaxSizeMB + + Comment + The maximum number of MB to use for the new disk cache + Persist + 1 + Type + U32 + Value + 10 + + DiskCacheDirName + + Comment + The name of the disk cache (within the standard Viewer disk cache directory) + Persist + 1 + Type + String + Value + dcache + CacheLocation Comment diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 0d25cb6dc8..1914ca89de 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -98,6 +98,7 @@ #include "lllogininstance.h" #include "llprogressview.h" #include "llvocache.h" +#include "lldiskcache.h" #include "llvopartgroup.h" #include "llweb.h" #include "llfloatertexturefetchdebugger.h" @@ -115,7 +116,6 @@ #include "llprimitive.h" #include "llurlaction.h" #include "llurlentry.h" -#include "llfilesystem.h" #include "llvolumemgr.h" #include "llxfermanager.h" #include "llphysicsextensions.h" @@ -1855,9 +1855,6 @@ bool LLAppViewer::cleanup() SUBSYSTEM_CLEANUP(LLWorldMapView); SUBSYSTEM_CLEANUP(LLFolderViewItem); - LL_INFOS() << "Cleaning up disk cache" << LL_ENDL; - SUBSYSTEM_CLEANUP(LLFileSystem); - LL_INFOS() << "Saving Data" << LL_ENDL; // Store the time of our current logoff @@ -4111,6 +4108,9 @@ bool LLAppViewer::initCache() { LLSplashScreen::update(LLTrans::getString("StartupClearingCache")); purgeCache(); + + // purge the new C++ file system based cache + LLDiskCache::getInstance()->purge(); } LLSplashScreen::update(LLTrans::getString("StartupInitializingTextureCache")); @@ -4131,7 +4131,12 @@ bool LLAppViewer::initCache() LLVOCache::getInstance()->initCache(LL_PATH_CACHE, gSavedSettings.getU32("CacheNumberOfRegionsForObjects"), getObjectCacheVersion()); - LLFileSystem::initClass(); + // initialize the new disk cache using saved settings + const std::string cache_dir_name = gSavedSettings.getString("DiskCacheDirName"); + const unsigned int cache_max_bytes = gSavedSettings.getU32("DiskCacheMaxSizeMB") * 1024 * 1024; + const bool enable_cache_debug_info = gSavedSettings.getBOOL("EnableDiskCacheDebugInfo"); + const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, cache_dir_name); + LLDiskCache::initParamSingleton(cache_dir, cache_max_bytes, enable_cache_debug_info); return true; } -- cgit v1.3 From 5fdc4a1fb4a1788b20ca7a2e7764fd1391c35785 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 7 Oct 2020 16:03:28 -0700 Subject: Add in some cache stats for the about box --- indra/llfilesystem/lldiskcache.cpp | 20 ++++++++++++++++++-- indra/llfilesystem/lldiskcache.h | 22 +++++----------------- indra/newview/llappviewer.cpp | 3 +++ indra/newview/skins/default/xui/en/strings.xml | 3 ++- 4 files changed, 28 insertions(+), 20 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 4b82cf3cce..455e27221e 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -187,8 +187,6 @@ const std::string LLDiskCache::metaDataToFilepath(const std::string id, file_path << assetTypeToString(at); file_path << ".asset"; - LL_INFOS() << "filepath.str() = " << file_path.str() << LL_ENDL; - return file_path.str(); } @@ -203,6 +201,24 @@ void LLDiskCache::updateFileAccessTime(const std::string file_path) boost::filesystem::last_write_time(file_path, file_time); } +/** + * + */ +const std::string LLDiskCache::getCacheInfo() +{ + std::ostringstream cache_info; + + F32 max_in_mb = (F32)mMaxSizeBytes / (1024.0 * 1024.0); + F32 percent_used = ((F32)dirFileSize(mCacheDir) / (F32)mMaxSizeBytes) * 100.0; + + cache_info << std::fixed; + cache_info << std::setprecision(1); + cache_info << "Max size " << max_in_mb << " MB "; + cache_info << "(" << percent_used << "% used)"; + + return cache_info.str(); +} + /** * Clear the cache by removing all the files in the cache directory * individually. It's important to maintain control of which directory diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 12599a132e..34a4fda756 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -40,23 +40,6 @@ class LLDiskCache : virtual ~LLDiskCache() = default; public: - ///** - // * The location of the cache dir is set in llappviewer at startup via the - // * saved settings parameters. We do not have access to those saved settings - // * here in LLCommon so we must provide an accessor for other classes to use - // * when they need it - e.g. LLFilesystem needs the path so it can write files - // * to it. - // */ - //const std::string getCacheDirName() { return mCacheDir; } - - ///** - // * Each cache filename has a prefix inserted (see definition of the - // * mCacheFilenamePrefix variable below for why) but the LLFileSystem - // * component needs access to it to in order to create the file so we - // * expose it by a getter here. - // */ - //const std::string getCacheFilenamePrefix() { return mCacheFilenamePrefix; } - /** * Construct a filename and path to it based on the file meta data * (id, asset type, additional 'extra' info like discard level perhaps) @@ -86,6 +69,11 @@ class LLDiskCache : */ void clearCache(const std::string cache_dir); + /** + * Return some information about the cache for use in About Box etc. + */ + const std::string getCacheInfo(); + private: /** * Utility function to gather the total size the files in a given diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 1914ca89de..afafedf207 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -3196,6 +3196,9 @@ LLSD LLAppViewer::getViewerInfo() const info["SERVER_RELEASE_NOTES_URL"] = mServerReleaseNotesURL; } + // populate field for new local disk cache with some details + info["DISK_CACHE_INFO"] = LLDiskCache::getInstance()->getCacheInfo(); + return info; } diff --git a/indra/newview/skins/default/xui/en/strings.xml b/indra/newview/skins/default/xui/en/strings.xml index 03aed8aa7e..619c869a3d 100644 --- a/indra/newview/skins/default/xui/en/strings.xml +++ b/indra/newview/skins/default/xui/en/strings.xml @@ -55,8 +55,9 @@ LOD factor: [LOD_FACTOR] Render quality: [RENDER_QUALITY] Advanced Lighting Model: [GPU_SHADERS] Texture memory: [TEXTURE_MEMORY]MB +Disk cache: [DISK_CACHE_INFO] - + HiDPI display mode: [HIDPI] -- cgit v1.3 From 1a9942a51c2b5db51adb75356f342665743d1f16 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 7 Oct 2020 16:43:01 -0700 Subject: Improve, rationalize and expand comments --- indra/llfilesystem/lldiskcache.cpp | 49 +++++++++++++--------------- indra/llfilesystem/lldiskcache.h | 65 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 30 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index 455e27221e..e2e50c775d 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -2,6 +2,12 @@ * @file lldiskcache.cpp * @brief The disk cache implementation. * + * Note: Rather than keep the top level function comments up + * to date in both the source and header files, I elected to + * only have explicit comments about each function and variable + * in the header - look there for details. The same is true for + * description of how this code is supposed to work. + * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code * Copyright (C) 2020, Linden Research, Inc. @@ -40,10 +46,8 @@ LLDiskCache::LLDiskCache(const std::string cache_dir, mMaxSizeBytes(max_size_bytes), mEnableCacheDebugInfo(enable_cache_debug_info) { - // the prefix used for cache filenames to disambiguate them from other files mCacheFilenamePrefix = "sl_cache"; - // create cache dir if it does not exist boost::filesystem::create_directory(cache_dir); } @@ -126,7 +130,7 @@ void LLDiskCache::purge() const std::string LLDiskCache::assetTypeToString(LLAssetType::EType at) { /** - * Make use of the C++17 (or is it 14) feature that allows + * Make use of the handy C++17 feature that allows * for inline initialization of an std::map<> */ typedef std::map asset_type_to_name_t; @@ -190,20 +194,12 @@ const std::string LLDiskCache::metaDataToFilepath(const std::string id, return file_path.str(); } -/** - * Update the "last write time" of a file to "now". This must be called whenever a - * file in the cache is read (not written) so that the last time the file was - * accessed which is used in the mechanism for purging the cache, is up to date. - */ void LLDiskCache::updateFileAccessTime(const std::string file_path) { const std::time_t file_time = std::time(nullptr); boost::filesystem::last_write_time(file_path, file_time); } -/** - * - */ const std::string LLDiskCache::getCacheInfo() { std::ostringstream cache_info; @@ -219,14 +215,14 @@ const std::string LLDiskCache::getCacheInfo() return cache_info.str(); } -/** - * Clear the cache by removing all the files in the cache directory - * individually. It's important to maintain control of which directory - * if passed in and not let the user inadvertently (or maliciously) set - * it to an random location like your project source or OS system directory - */ void LLDiskCache::clearCache(const std::string cache_dir) { + /** + * See notes on performance in dirFileSize(..) - there may be + * a quicker way to do this by operating on the parent dir vs + * the component files but it's called infrequently so it's + * likely just fine + */ if (boost::filesystem::is_directory(cache_dir)) { for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_dir), {})) @@ -242,20 +238,19 @@ void LLDiskCache::clearCache(const std::string cache_dir) } } -/** - * Utility function to get the total filesize of all files in a directory. It - * used to test file extensions to only check cache files but that was removed. - * There may be a better way that works directly on the folder (similar to - * right clicking on a folder in the OS and asking for size vs right clicking - * on all files and adding up manually) but this is very fast - less than 100ms - * in my testing so, so long as it's not called frequently, it should be okay. - * Note that's it's only currently used for logging/debugging so if performance - * is ever an issue, optimizing this or removing it altogether, is an easy win. - */ uintmax_t LLDiskCache::dirFileSize(const std::string dir) { uintmax_t total_file_size = 0; + /** + * There may be a better way that works directly on the folder (similar to + * right clicking on a folder in the OS and asking for size vs right clicking + * on all files and adding up manually) but this is very fast - less than 100ms + * for 10,000 files in my testing so, so long as it's not called frequently, + * it should be okay. Note that's it's only currently used for logging/debugging + * so if performance is ever an issue, optimizing this or removing it altogether, + * is an easy win. + */ if (boost::filesystem::is_directory(dir)) { for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(dir), {})) diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 34a4fda756..f718b7a328 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -1,6 +1,41 @@ /** * @file lldiskcache.h - * @brief The disk cache implementation. + * @brief The disk cache implementation declarations. + * + * @Description: + * This code implements a disk cache using the following ideas: + * 1/ The metadata for a file can be encapsulated in the filename. + The filenames will be composed of the following fields: + Prefix: Used to identify the file as a part of the cache. + An additional reason for using a prefix is that it + might be possible, either accidentally or maliciously + to end up with the cache dir set to a non-cache + location such as your OS system dir or a work folder. + Purging files from that would obviously be a disaster + so this is an extra step to help avoid that scenario. + ID: Typically the asset ID (UUID) of the asset being + saved but can be anything valid for a filename + Extra Info: A field for use in the future that can be used + to store extra identifiers - e.g. the discard + level of a JPEG2000 file + Asset Type: A text string created from the LLAssetType enum + that identifies the type of asset being stored. + .asset A file extension of .asset is used to help + identify this as a Viewer asset file + * 2/ The time of last access for a file can be updated instantly + * for file reads and automatically as part of the file writes. + * 3/ The purge algorithm collects a list of all files in the + * directory, sorts them by date of last access (write) and then + * deletes any files based on age until the total size of all + * the files is less than the maximum size specified. + * 4/ An LLSingleton idiom is used since there will only ever be + * a single cache and we want to access it from numerous places. + * 5/ Performance on my modest system seems very acceptable. For + * example, in testing, I was able to purge a directory of + * 10,000 files, deleting about half of them in ~ 1700ms. For + * the same sized directory of files, writing the last updated + * time to each took less than 600ms indicating that this + * important part of the mechanism has almost no overhead. * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code @@ -33,10 +68,32 @@ class LLDiskCache : public LLParamSingleton { public: + /** + * Since this is using the LLSingleton pattern but we + * want to allow the constructor to be called first + * with various parameters, we also invoke the + * LLParamSingleton idiom and use it to initialize + * the class via a call in LLAppViewer. + */ LLSINGLETON(LLDiskCache, + /** + * The full name of the cache folder - typically a + * a child of the main Viewer cache directory. Defined + * by the setting at 'DiskCacheDirName' + */ const std::string cache_dir, + /** + * The maximum size of the cache in bytes - Defined by + * the setting at 'DiskCacheMaxSizeMB' (* 1024 * 1024) + */ const int max_size_bytes, + /** + * A flag that enables extra cache debugging so that + * if there are bugs, we can ask uses to enable this + * setting and send us their logs + */ const bool enable_cache_debug_info); + virtual ~LLDiskCache() = default; public: @@ -54,7 +111,7 @@ class LLDiskCache : /** * Update the "last write time" of a file to "now". This must be called whenever a * file in the cache is read (not written) so that the last time the file was - * accessed which is used in the mechanism for purging the cache, is up to date. + * accessed is up to date (This is used in the mechanism for purging the cache) */ void updateFileAccessTime(const std::string file_path); @@ -65,7 +122,9 @@ class LLDiskCache : void purge(); /** - * Clear the cache by removing the files in the cache directory individually + * Clear the cache by removing all the files in the specified cache + * directory individually. Only the files that contain a prefix defined + * by mCacheFilenamePrefix will be removed. */ void clearCache(const std::string cache_dir); -- cgit v1.3 From 3051db7b61ee43fffd28f0a12c0714b11b6b7df7 Mon Sep 17 00:00:00 2001 From: Mnikolenko Productengine Date: Thu, 8 Oct 2020 15:26:54 +0300 Subject: Purge excessive files from disc cache each startup --- indra/llfilesystem/lldiskcache.cpp | 6 +++--- indra/llfilesystem/lldiskcache.h | 2 +- indra/newview/llappviewer.cpp | 32 ++++++++++++++++++++------------ 3 files changed, 24 insertions(+), 16 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.cpp b/indra/llfilesystem/lldiskcache.cpp index e2e50c775d..efe5e7092c 100644 --- a/indra/llfilesystem/lldiskcache.cpp +++ b/indra/llfilesystem/lldiskcache.cpp @@ -215,7 +215,7 @@ const std::string LLDiskCache::getCacheInfo() return cache_info.str(); } -void LLDiskCache::clearCache(const std::string cache_dir) +void LLDiskCache::clearCache() { /** * See notes on performance in dirFileSize(..) - there may be @@ -223,9 +223,9 @@ void LLDiskCache::clearCache(const std::string cache_dir) * the component files but it's called infrequently so it's * likely just fine */ - if (boost::filesystem::is_directory(cache_dir)) + if (boost::filesystem::is_directory(mCacheDir)) { - for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(cache_dir), {})) + for (auto& entry : boost::make_iterator_range(boost::filesystem::directory_iterator(mCacheDir), {})) { if (boost::filesystem::is_regular_file(entry)) { diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index f718b7a328..b25eac8538 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -126,7 +126,7 @@ class LLDiskCache : * directory individually. Only the files that contain a prefix defined * by mCacheFilenamePrefix will be removed. */ - void clearCache(const std::string cache_dir); + void clearCache(); /** * Return some information about the cache for use in About Box etc. diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index afafedf207..9e50860064 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -4061,6 +4061,13 @@ bool LLAppViewer::initCache() LLAppViewer::getTextureCache()->setReadOnly(read_only) ; LLVOCache::initParamSingleton(read_only); + // initialize the new disk cache using saved settings + const std::string cache_dir_name = gSavedSettings.getString("DiskCacheDirName"); + const unsigned int cache_max_bytes = gSavedSettings.getU32("DiskCacheMaxSizeMB") * 1024 * 1024; + const bool enable_cache_debug_info = gSavedSettings.getBOOL("EnableDiskCacheDebugInfo"); + const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, cache_dir_name); + LLDiskCache::initParamSingleton(cache_dir, cache_max_bytes, enable_cache_debug_info); + bool texture_cache_mismatch = false; if (gSavedSettings.getS32("LocalCacheVersion") != LLAppViewer::getTextureCacheVersion()) { @@ -4107,13 +4114,21 @@ bool LLAppViewer::initCache() gSavedSettings.setString("CacheLocationTopFolder", ""); } - if (mPurgeCache && !read_only) + if (!read_only) { - LLSplashScreen::update(LLTrans::getString("StartupClearingCache")); - purgeCache(); + if (mPurgeCache) + { + LLSplashScreen::update(LLTrans::getString("StartupClearingCache")); + purgeCache(); - // purge the new C++ file system based cache - LLDiskCache::getInstance()->purge(); + // clear the new C++ file system based cache + LLDiskCache::getInstance()->clearCache(); + } + else + { + // purge excessive files from the new file system based cache + LLDiskCache::getInstance()->purge(); + } } LLSplashScreen::update(LLTrans::getString("StartupInitializingTextureCache")); @@ -4134,13 +4149,6 @@ bool LLAppViewer::initCache() LLVOCache::getInstance()->initCache(LL_PATH_CACHE, gSavedSettings.getU32("CacheNumberOfRegionsForObjects"), getObjectCacheVersion()); - // initialize the new disk cache using saved settings - const std::string cache_dir_name = gSavedSettings.getString("DiskCacheDirName"); - const unsigned int cache_max_bytes = gSavedSettings.getU32("DiskCacheMaxSizeMB") * 1024 * 1024; - const bool enable_cache_debug_info = gSavedSettings.getBOOL("EnableDiskCacheDebugInfo"); - const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, cache_dir_name); - LLDiskCache::initParamSingleton(cache_dir, cache_max_bytes, enable_cache_debug_info); - return true; } -- cgit v1.3 From 391ada1150a861e899dcde8558e9efd4c5efc981 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 27 Oct 2020 13:53:17 -0700 Subject: Fix for meta issue: SL-14211 Determine optimum cache size for VFS replacement cache --- indra/llfilesystem/lldiskcache.h | 4 ++-- indra/newview/app_settings/settings.xml | 8 ++++---- indra/newview/llappviewer.cpp | 10 ++++++++-- 3 files changed, 14 insertions(+), 8 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index b25eac8538..997884da31 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -83,8 +83,8 @@ class LLDiskCache : */ const std::string cache_dir, /** - * The maximum size of the cache in bytes - Defined by - * the setting at 'DiskCacheMaxSizeMB' (* 1024 * 1024) + * The maximum size of the cache in bytes - Based on the + * setting at 'CacheSize' and 'DiskCachePercentOfTotal' */ const int max_size_bytes, /** diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index fc1437148a..142a3098ec 100644 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -1362,16 +1362,16 @@ Value 0 - DiskCacheMaxSizeMB + DiskCachePercentOfTotal Comment - The maximum number of MB to use for the new disk cache + The percent of total cache size (defined by CacheSize) to use for the disk cache Persist 1 Type - U32 + F32 Value - 50 + 20.0 DiskCacheDirName diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 9e50860064..20ca432279 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -4063,10 +4063,16 @@ bool LLAppViewer::initCache() // initialize the new disk cache using saved settings const std::string cache_dir_name = gSavedSettings.getString("DiskCacheDirName"); - const unsigned int cache_max_bytes = gSavedSettings.getU32("DiskCacheMaxSizeMB") * 1024 * 1024; + + // note that the maximum size of this cache is defined as a percentage of the + // total cache size - the 'CacheSize' pref - for all caches. + const unsigned int cache_total_size_mb = gSavedSettings.getU32("CacheSize"); + const double disk_cache_percent = gSavedSettings.getF32("DiskCachePercentOfTotal"); + const unsigned int disk_cache_mb = cache_total_size_mb * disk_cache_percent / 100; + const unsigned int disk_cache_bytes = disk_cache_mb * 1024 * 1024; const bool enable_cache_debug_info = gSavedSettings.getBOOL("EnableDiskCacheDebugInfo"); const std::string cache_dir = gDirUtilp->getExpandedFilename(LL_PATH_CACHE, cache_dir_name); - LLDiskCache::initParamSingleton(cache_dir, cache_max_bytes, enable_cache_debug_info); + LLDiskCache::initParamSingleton(cache_dir, disk_cache_bytes, enable_cache_debug_info); bool texture_cache_mismatch = false; if (gSavedSettings.getS32("LocalCacheVersion") != LLAppViewer::getTextureCacheVersion()) -- cgit v1.3