From d9448c6f52218146113d1d5c5ca4c4d5f01dc5cf Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Thu, 17 Sep 2020 09:45:06 -0700 Subject: The folder where the disk cache lives was originally renamed from llvfs to llcache but @henri's suggestion that that doesn't reflect the other files in the same place and it should be llfilesystem is a good one so I changed it over --- indra/llfilesystem/lldiskcache.h | 82 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 indra/llfilesystem/lldiskcache.h (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h new file mode 100644 index 0000000000..7ad06a8689 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.h @@ -0,0 +1,82 @@ +/** + * @file lldiskcacke.h + * @brief Definition of virtual file + * + * $LicenseInfo:firstyear=2002&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2010, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLDISKCACHE_H +#define LL_LLDISKCACHE_H + +#include "lluuid.h" +#include "llassettype.h" + +class LLDiskCache +{ +public: + LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLDiskCache::READ); + ~LLDiskCache(); + + BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ + BOOL isReadComplete(); + S32 getLastBytesRead(); + BOOL eof(); + + BOOL write(const U8 *buffer, S32 bytes); + static BOOL writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type); + BOOL seek(S32 offset, S32 origin = -1); + S32 tell() const; + + S32 getSize(); + S32 getMaxSize(); + BOOL setMaxSize(S32 size); + BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); + BOOL remove(); + + bool isLocked(); + void waitForLock(); + + static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); + static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); + static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, + const LLUUID &new_file_id, const LLAssetType::EType new_file_type); + static S32 getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type); + + static void initClass(); + static void cleanupClass(); + +public: + static const S32 READ; + static const S32 WRITE; + static const S32 READ_WRITE; + static const S32 APPEND; + +protected: + LLAssetType::EType mFileType; + BOOL mReadComplete; + LLUUID mFileID; + S32 mPosition; + S32 mMode; + S32 mBytesRead; +}; + +#endif // LL_LLDISKCACHE_H -- cgit v1.2.3 From 96e2873bfa2c6b8823aed3b4190c43cd5dab54e6 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Thu, 24 Sep 2020 10:23:39 -0700 Subject: Rename lldiskcache.* to llfilesystem.* - i think this is the right name since it's responsible for performing file system operations and (will eventually) delegrate to a separate disk cache component to save/load data and keep track of metadata etc. --- indra/llfilesystem/lldiskcache.h | 82 ---------------------------------------- 1 file changed, 82 deletions(-) delete mode 100644 indra/llfilesystem/lldiskcache.h (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h deleted file mode 100644 index 7ad06a8689..0000000000 --- a/indra/llfilesystem/lldiskcache.h +++ /dev/null @@ -1,82 +0,0 @@ -/** - * @file lldiskcacke.h - * @brief Definition of virtual file - * - * $LicenseInfo:firstyear=2002&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#ifndef LL_LLDISKCACHE_H -#define LL_LLDISKCACHE_H - -#include "lluuid.h" -#include "llassettype.h" - -class LLDiskCache -{ -public: - LLDiskCache(const LLUUID &file_id, const LLAssetType::EType file_type, S32 mode = LLDiskCache::READ); - ~LLDiskCache(); - - BOOL read(U8 *buffer, S32 bytes, BOOL async = FALSE, F32 priority = 128.f); /* Flawfinder: ignore */ - BOOL isReadComplete(); - S32 getLastBytesRead(); - BOOL eof(); - - BOOL write(const U8 *buffer, S32 bytes); - static BOOL writeFile(const U8 *buffer, S32 bytes, const LLUUID &uuid, LLAssetType::EType type); - BOOL seek(S32 offset, S32 origin = -1); - S32 tell() const; - - S32 getSize(); - S32 getMaxSize(); - BOOL setMaxSize(S32 size); - BOOL rename(const LLUUID &new_id, const LLAssetType::EType new_type); - BOOL remove(); - - bool isLocked(); - void waitForLock(); - - static bool getExists(const LLUUID &file_id, const LLAssetType::EType file_type); - static bool removeFile(const LLUUID &file_id, const LLAssetType::EType file_type); - static bool renameFile(const LLUUID &old_file_id, const LLAssetType::EType old_file_type, - const LLUUID &new_file_id, const LLAssetType::EType new_file_type); - static S32 getFileSize(const LLUUID &file_id, const LLAssetType::EType file_type); - - static void initClass(); - static void cleanupClass(); - -public: - static const S32 READ; - static const S32 WRITE; - static const S32 READ_WRITE; - static const S32 APPEND; - -protected: - LLAssetType::EType mFileType; - BOOL mReadComplete; - LLUUID mFileID; - S32 mPosition; - S32 mMode; - S32 mBytesRead; -}; - -#endif // LL_LLDISKCACHE_H -- cgit v1.2.3 From 6be1f88a5ef99e1e40bb5701a250ba0728f56005 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Thu, 24 Sep 2020 14:45:39 -0700 Subject: Complete the change from lldiskcache -> llfilesystem and then addition of new lldiskcache implementation --- indra/llfilesystem/lldiskcache.h | 130 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 indra/llfilesystem/lldiskcache.h (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h new file mode 100644 index 0000000000..39b8f7ef48 --- /dev/null +++ b/indra/llfilesystem/lldiskcache.h @@ -0,0 +1,130 @@ +/** + * @file lldiskcache.h + * @brief Declaration SQLite meta data / file storage API + * @brief Declaration of the generic disk cache interface + as well the SQLite header/implementation. + * @Note Eventually, this component might split into an interface + * file and multiple implemtations but for now, this is the + * only one so I think it's okay to combine everything. + * + * $LicenseInfo:firstyear=2009&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2020, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLDISKCACHE +#define _LLDISKCACHE + +#include +#include + +#include "sqlite3.h" + +// Enable this line to display each SQL statement when it is executed +// It can lead to a lot of spam but useful for debugging +//#define SHOW_STATEMENTS + +// I toyed with the idea of a variety of approaches and thought have +// an abstract base class with which to hang different implementations +// off was a good idea but I think we settled on a single approach +// so I will likely remove this level of indirection if not other +// interesting implementation ideas present themselves. +class llDiskCacheBase +{ + public: + llDiskCacheBase() {}; + virtual ~llDiskCacheBase() {}; + virtual bool open(const std::string db_folder, + const std::string db_filename) = 0; + virtual bool exists(const std::string id, bool& exists) = 0; + virtual bool put(const std::string id, + char* binary_data, + int binary_data_size) = 0; + virtual const bool get(const std::string id, + char** mem_buffer, + int& mem_buffer_size) = 0; + virtual bool purge(int num_entries) = 0; + virtual void close() = 0; + virtual const std::string dbVersion() = 0; + virtual const std::string getFilenameById(const std::string id) = 0; + virtual const int getAccessCountById(const std::string id) = 0; + virtual void getNumEntries(int& num_entries, int& max_entries) = 0; +}; + +// implementation for the SQLite/disk blended case +// see source file for detailed comments +class llDiskCache : + public llDiskCacheBase +{ + public: + llDiskCache(); + virtual ~llDiskCache(); + virtual bool open(const std::string db_folder, + const std::string db_filename) override; + virtual bool exists(const std::string id, bool& exists) override; + virtual bool put(const std::string id, + char* binary_data, + int binary_data_size) override; + virtual const bool get(const std::string id, + char** mem_buffer, + int& mem_buffer_size) override; + virtual bool purge(int num_entries) override; + virtual void close() override; + virtual const std::string dbVersion() override; + virtual const std::string getFilenameById(const std::string id) override; + virtual const int getAccessCountById(const std::string id) override; + virtual void getNumEntries(int& num_entries, int& max_entries) override; + + private: + sqlite3* mDb; + std::string mDataStorePath; + const std::string mTableName = "lldiskcache"; + const std::string mIdFieldName = "id"; + const std::string mFilenameFieldName = "filename"; + const std::string mFilesizeFieldName = "filesize"; + const std::string mInsertionDateTimeFieldName = "insert_datetime"; + const std::string mLastAccessDateTimeFieldName = "last_access_datetime"; + const std::string mAccessCountFieldName = "access_count"; + + private: + void printError(const std::string funcname, + const std::string desc, + bool is_sqlite_err); + bool beginTransaction(); + bool endTransaction(); + std::string makeUniqueFilename(); + const std::string systemSeparator(); + const std::string makeFullPath(const std::string filename); + bool sqliteExec(const std::string stmt, + const std::string funcname); + sqlite3_stmt* sqlitePrepareStep(const std::string funcname, + const std::string stmt, + int sqlite_success_condition); + const std::string sqlComposeCreateTable(); + const std::string sqlComposeExists(const std::string id); + const std::string sqlComposePut(const std::string id, + const std::string filename, + int binary_data_size); + const std::string sqlComposeGetSelect(const std::string id); + const std::string sqlComposeGetUpdate(const std::string id); + const std::string sqlComposePurge(int num_entries); +}; + +#endif // _LLDISKCACHE -- cgit v1.2.3 From 3092aa8aae496803707980eb456cddbb9960ef1c Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 6 Oct 2020 17:16:53 -0700 Subject: Add in the C++ filesystem based cache and clean up some indempotent functions in llfilesystem --- indra/llfilesystem/lldiskcache.h | 146 +++++++++++++++------------------------ 1 file changed, 55 insertions(+), 91 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 39b8f7ef48..1618cec6a6 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -1,11 +1,6 @@ /** * @file lldiskcache.h - * @brief Declaration SQLite meta data / file storage API - * @brief Declaration of the generic disk cache interface - as well the SQLite header/implementation. - * @Note Eventually, this component might split into an interface - * file and multiple implemtations but for now, this is the - * only one so I think it's okay to combine everything. + * @brief The disk cache implementation. * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code @@ -32,99 +27,68 @@ #ifndef _LLDISKCACHE #define _LLDISKCACHE -#include -#include +class LLDiskCache +{ + public: + LLDiskCache(const std::string cache_dir); + ~LLDiskCache(); -#include "sqlite3.h" + /** + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was + * accessed which is used in the mechanism for purging the cache, is up to date. + */ + void updateFileAccessTime(const std::string file_path); -// Enable this line to display each SQL statement when it is executed -// It can lead to a lot of spam but useful for debugging -//#define SHOW_STATEMENTS + /** + * Purge the oldest items in the cache so that the combined size of all files + * is no bigger than mMaxSizeBytes. + */ + void purge(); -// I toyed with the idea of a variety of approaches and thought have -// an abstract base class with which to hang different implementations -// off was a good idea but I think we settled on a single approach -// so I will likely remove this level of indirection if not other -// interesting implementation ideas present themselves. -class llDiskCacheBase -{ - public: - llDiskCacheBase() {}; - virtual ~llDiskCacheBase() {}; - virtual bool open(const std::string db_folder, - const std::string db_filename) = 0; - virtual bool exists(const std::string id, bool& exists) = 0; - virtual bool put(const std::string id, - char* binary_data, - int binary_data_size) = 0; - virtual const bool get(const std::string id, - char** mem_buffer, - int& mem_buffer_size) = 0; - virtual bool purge(int num_entries) = 0; - virtual void close() = 0; - virtual const std::string dbVersion() = 0; - virtual const std::string getFilenameById(const std::string id) = 0; - virtual const int getAccessCountById(const std::string id) = 0; - virtual void getNumEntries(int& num_entries, int& max_entries) = 0; -}; + /** + * Clear the cache by removing the files in the cache directory individually + */ + void clearCache(const std::string cache_dir); -// implementation for the SQLite/disk blended case -// see source file for detailed comments -class llDiskCache : - public llDiskCacheBase -{ - public: - llDiskCache(); - virtual ~llDiskCache(); - virtual bool open(const std::string db_folder, - const std::string db_filename) override; - virtual bool exists(const std::string id, bool& exists) override; - virtual bool put(const std::string id, - char* binary_data, - int binary_data_size) override; - virtual const bool get(const std::string id, - char** mem_buffer, - int& mem_buffer_size) override; - virtual bool purge(int num_entries) override; - virtual void close() override; - virtual const std::string dbVersion() override; - virtual const std::string getFilenameById(const std::string id) override; - virtual const int getAccessCountById(const std::string id) override; - virtual void getNumEntries(int& num_entries, int& max_entries) override; + /** + * Manage max size in bytes of cache - use a discrete setter/getter so the value can + * be changed in the preferences and cache cleared/purged without restarting viewer + * to instantiate this class again. + */ + void setMaxSizeBytes(const uintmax_t size_bytes) { mMaxSizeBytes = size_bytes; } + uintmax_t getMaxSizeBytes() const { return mMaxSizeBytes; } private: - sqlite3* mDb; - std::string mDataStorePath; - const std::string mTableName = "lldiskcache"; - const std::string mIdFieldName = "id"; - const std::string mFilenameFieldName = "filename"; - const std::string mFilesizeFieldName = "filesize"; - const std::string mInsertionDateTimeFieldName = "insert_datetime"; - const std::string mLastAccessDateTimeFieldName = "last_access_datetime"; - const std::string mAccessCountFieldName = "access_count"; + /** + * Utility function to gather the total size the files in a given + * directory. Primarily used here to determine the directory size + * before and after the cache purge + */ + uintmax_t dirFileSize(const std::string dir); private: - void printError(const std::string funcname, - const std::string desc, - bool is_sqlite_err); - bool beginTransaction(); - bool endTransaction(); - std::string makeUniqueFilename(); - const std::string systemSeparator(); - const std::string makeFullPath(const std::string filename); - bool sqliteExec(const std::string stmt, - const std::string funcname); - sqlite3_stmt* sqlitePrepareStep(const std::string funcname, - const std::string stmt, - int sqlite_success_condition); - const std::string sqlComposeCreateTable(); - const std::string sqlComposeExists(const std::string id); - const std::string sqlComposePut(const std::string id, - const std::string filename, - int binary_data_size); - const std::string sqlComposeGetSelect(const std::string id); - const std::string sqlComposeGetUpdate(const std::string id); - const std::string sqlComposePurge(int num_entries); + /** + * Default of 20MB seems reasonable - it will likely be reset + * immediately anyway using a value from the Viewer settings + */ + const uintmax_t mDefaultSizeBytes = 20 * 1024 * 1024; + uintmax_t mMaxSizeBytes; + + /** + * The folder that holds the cached files. The consumer of this + * class must avoid letting the user set this location as a malicious + * setting could potentially point it at a non-cache directory (for example, + * the Windows System dir) with disastrous results. + */ + std::string mCacheDir; + + /** + * This is set from the CacheDebugInfo global setting and + * when enabled, displays additional debugging information in + * various parts of the code + */ + bool mCacheDebugInfo; }; #endif // _LLDISKCACHE -- cgit v1.2.3 From 08dfc0836fb12855d0c07d811e2909400d5b74f3 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 7 Oct 2020 15:25:12 -0700 Subject: This changeset hooks up many things that have been in progress and moves things about between llfilesystem and lldiskcache - there is still some bookkeeping work left but this is the first version that appears to work and actively manage the cache --- indra/llfilesystem/lldiskcache.h | 82 ++++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 20 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 1618cec6a6..12599a132e 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -27,15 +27,50 @@ #ifndef _LLDISKCACHE #define _LLDISKCACHE -class LLDiskCache +#include "llsingleton.h" + +class LLDiskCache : + public LLParamSingleton { public: - LLDiskCache(const std::string cache_dir); - ~LLDiskCache(); + LLSINGLETON(LLDiskCache, + const std::string cache_dir, + const int max_size_bytes, + const bool enable_cache_debug_info); + virtual ~LLDiskCache() = default; + + public: + ///** + // * The location of the cache dir is set in llappviewer at startup via the + // * saved settings parameters. We do not have access to those saved settings + // * here in LLCommon so we must provide an accessor for other classes to use + // * when they need it - e.g. LLFilesystem needs the path so it can write files + // * to it. + // */ + //const std::string getCacheDirName() { return mCacheDir; } + + ///** + // * Each cache filename has a prefix inserted (see definition of the + // * mCacheFilenamePrefix variable below for why) but the LLFileSystem + // * component needs access to it to in order to create the file so we + // * expose it by a getter here. + // */ + //const std::string getCacheFilenamePrefix() { return mCacheFilenamePrefix; } /** - * Update the "last write time" of a file to "now". This must be called whenever a - * file in the cache is read (not written) so that the last time the file was + * Construct a filename and path to it based on the file meta data + * (id, asset type, additional 'extra' info like discard level perhaps) + * Worth pointing out that this function used to be in LLFileSystem but + * so many things had to be pushed back there to accomodate it, that I + * decided to move it here. Still not sure that's completely right. + */ + const std::string metaDataToFilepath(const std::string id, + LLAssetType::EType at, + const std::string extra_info); + + /** + * Update the "last write time" of a file to "now". This must be called whenever a + * file in the cache is read (not written) so that the last time the file was * accessed which is used in the mechanism for purging the cache, is up to date. */ void updateFileAccessTime(const std::string file_path); @@ -51,28 +86,26 @@ class LLDiskCache */ void clearCache(const std::string cache_dir); - /** - * Manage max size in bytes of cache - use a discrete setter/getter so the value can - * be changed in the preferences and cache cleared/purged without restarting viewer - * to instantiate this class again. - */ - void setMaxSizeBytes(const uintmax_t size_bytes) { mMaxSizeBytes = size_bytes; } - uintmax_t getMaxSizeBytes() const { return mMaxSizeBytes; } - private: /** * Utility function to gather the total size the files in a given - * directory. Primarily used here to determine the directory size + * directory. Primarily used here to determine the directory size * before and after the cache purge */ uintmax_t dirFileSize(const std::string dir); + /** + * Utility function to convert an LLAssetType enum into a + * string that we use as part of the cache file filename + */ + const std::string assetTypeToString(LLAssetType::EType at); + private: /** - * Default of 20MB seems reasonable - it will likely be reset - * immediately anyway using a value from the Viewer settings + * The maximum size of the cache in bytes. After purge is called, the + * total size of the cache files in the cache directory will be + * less than this value */ - const uintmax_t mDefaultSizeBytes = 20 * 1024 * 1024; uintmax_t mMaxSizeBytes; /** @@ -84,11 +117,20 @@ class LLDiskCache std::string mCacheDir; /** - * This is set from the CacheDebugInfo global setting and - * when enabled, displays additional debugging information in + * The prefix inserted at the start of a cache file filename to + * help identify it as a cache file. It's probably not required + * (just the presence in the cache folder is enough) but I am + * paranoid about the cache folder being set to something bad + * like the users' OS system dir by mistake or maliciously and + * this will help to offset any damage if that happens. + */ + std::string mCacheFilenamePrefix; + + /** + * When enabled, displays additional debugging information in * various parts of the code */ - bool mCacheDebugInfo; + bool mEnableCacheDebugInfo; }; #endif // _LLDISKCACHE -- cgit v1.2.3 From 5fdc4a1fb4a1788b20ca7a2e7764fd1391c35785 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 7 Oct 2020 16:03:28 -0700 Subject: Add in some cache stats for the about box --- indra/llfilesystem/lldiskcache.h | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 12599a132e..34a4fda756 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -40,23 +40,6 @@ class LLDiskCache : virtual ~LLDiskCache() = default; public: - ///** - // * The location of the cache dir is set in llappviewer at startup via the - // * saved settings parameters. We do not have access to those saved settings - // * here in LLCommon so we must provide an accessor for other classes to use - // * when they need it - e.g. LLFilesystem needs the path so it can write files - // * to it. - // */ - //const std::string getCacheDirName() { return mCacheDir; } - - ///** - // * Each cache filename has a prefix inserted (see definition of the - // * mCacheFilenamePrefix variable below for why) but the LLFileSystem - // * component needs access to it to in order to create the file so we - // * expose it by a getter here. - // */ - //const std::string getCacheFilenamePrefix() { return mCacheFilenamePrefix; } - /** * Construct a filename and path to it based on the file meta data * (id, asset type, additional 'extra' info like discard level perhaps) @@ -86,6 +69,11 @@ class LLDiskCache : */ void clearCache(const std::string cache_dir); + /** + * Return some information about the cache for use in About Box etc. + */ + const std::string getCacheInfo(); + private: /** * Utility function to gather the total size the files in a given -- cgit v1.2.3 From 1a9942a51c2b5db51adb75356f342665743d1f16 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Wed, 7 Oct 2020 16:43:01 -0700 Subject: Improve, rationalize and expand comments --- indra/llfilesystem/lldiskcache.h | 65 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index 34a4fda756..f718b7a328 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -1,6 +1,41 @@ /** * @file lldiskcache.h - * @brief The disk cache implementation. + * @brief The disk cache implementation declarations. + * + * @Description: + * This code implements a disk cache using the following ideas: + * 1/ The metadata for a file can be encapsulated in the filename. + The filenames will be composed of the following fields: + Prefix: Used to identify the file as a part of the cache. + An additional reason for using a prefix is that it + might be possible, either accidentally or maliciously + to end up with the cache dir set to a non-cache + location such as your OS system dir or a work folder. + Purging files from that would obviously be a disaster + so this is an extra step to help avoid that scenario. + ID: Typically the asset ID (UUID) of the asset being + saved but can be anything valid for a filename + Extra Info: A field for use in the future that can be used + to store extra identifiers - e.g. the discard + level of a JPEG2000 file + Asset Type: A text string created from the LLAssetType enum + that identifies the type of asset being stored. + .asset A file extension of .asset is used to help + identify this as a Viewer asset file + * 2/ The time of last access for a file can be updated instantly + * for file reads and automatically as part of the file writes. + * 3/ The purge algorithm collects a list of all files in the + * directory, sorts them by date of last access (write) and then + * deletes any files based on age until the total size of all + * the files is less than the maximum size specified. + * 4/ An LLSingleton idiom is used since there will only ever be + * a single cache and we want to access it from numerous places. + * 5/ Performance on my modest system seems very acceptable. For + * example, in testing, I was able to purge a directory of + * 10,000 files, deleting about half of them in ~ 1700ms. For + * the same sized directory of files, writing the last updated + * time to each took less than 600ms indicating that this + * important part of the mechanism has almost no overhead. * * $LicenseInfo:firstyear=2009&license=viewerlgpl$ * Second Life Viewer Source Code @@ -33,10 +68,32 @@ class LLDiskCache : public LLParamSingleton { public: + /** + * Since this is using the LLSingleton pattern but we + * want to allow the constructor to be called first + * with various parameters, we also invoke the + * LLParamSingleton idiom and use it to initialize + * the class via a call in LLAppViewer. + */ LLSINGLETON(LLDiskCache, + /** + * The full name of the cache folder - typically a + * a child of the main Viewer cache directory. Defined + * by the setting at 'DiskCacheDirName' + */ const std::string cache_dir, + /** + * The maximum size of the cache in bytes - Defined by + * the setting at 'DiskCacheMaxSizeMB' (* 1024 * 1024) + */ const int max_size_bytes, + /** + * A flag that enables extra cache debugging so that + * if there are bugs, we can ask uses to enable this + * setting and send us their logs + */ const bool enable_cache_debug_info); + virtual ~LLDiskCache() = default; public: @@ -54,7 +111,7 @@ class LLDiskCache : /** * Update the "last write time" of a file to "now". This must be called whenever a * file in the cache is read (not written) so that the last time the file was - * accessed which is used in the mechanism for purging the cache, is up to date. + * accessed is up to date (This is used in the mechanism for purging the cache) */ void updateFileAccessTime(const std::string file_path); @@ -65,7 +122,9 @@ class LLDiskCache : void purge(); /** - * Clear the cache by removing the files in the cache directory individually + * Clear the cache by removing all the files in the specified cache + * directory individually. Only the files that contain a prefix defined + * by mCacheFilenamePrefix will be removed. */ void clearCache(const std::string cache_dir); -- cgit v1.2.3 From 3051db7b61ee43fffd28f0a12c0714b11b6b7df7 Mon Sep 17 00:00:00 2001 From: Mnikolenko Productengine Date: Thu, 8 Oct 2020 15:26:54 +0300 Subject: Purge excessive files from disc cache each startup --- indra/llfilesystem/lldiskcache.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index f718b7a328..b25eac8538 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -126,7 +126,7 @@ class LLDiskCache : * directory individually. Only the files that contain a prefix defined * by mCacheFilenamePrefix will be removed. */ - void clearCache(const std::string cache_dir); + void clearCache(); /** * Return some information about the cache for use in About Box etc. -- cgit v1.2.3 From 391ada1150a861e899dcde8558e9efd4c5efc981 Mon Sep 17 00:00:00 2001 From: Callum Prentice Date: Tue, 27 Oct 2020 13:53:17 -0700 Subject: Fix for meta issue: SL-14211 Determine optimum cache size for VFS replacement cache --- indra/llfilesystem/lldiskcache.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/llfilesystem/lldiskcache.h') diff --git a/indra/llfilesystem/lldiskcache.h b/indra/llfilesystem/lldiskcache.h index b25eac8538..997884da31 100644 --- a/indra/llfilesystem/lldiskcache.h +++ b/indra/llfilesystem/lldiskcache.h @@ -83,8 +83,8 @@ class LLDiskCache : */ const std::string cache_dir, /** - * The maximum size of the cache in bytes - Defined by - * the setting at 'DiskCacheMaxSizeMB' (* 1024 * 1024) + * The maximum size of the cache in bytes - Based on the + * setting at 'CacheSize' and 'DiskCachePercentOfTotal' */ const int max_size_bytes, /** -- cgit v1.2.3