From d484fff77d73931e70c896b9a88db547c4922dbd Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Tue, 25 Mar 2025 14:30:29 +0200 Subject: #3596 Don't supress exceptions in WorkQueue We need those to know about problems. --- indra/llcommon/workqueue.cpp | 83 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) (limited to 'indra/llcommon/workqueue.cpp') diff --git a/indra/llcommon/workqueue.cpp b/indra/llcommon/workqueue.cpp index 6066e74fb5..dace95aaf2 100644 --- a/indra/llcommon/workqueue.cpp +++ b/indra/llcommon/workqueue.cpp @@ -17,6 +17,7 @@ // std headers // external library headers // other Linden headers +#include "llapp.h" #include "llcoros.h" #include LLCOROS_MUTEX_HEADER #include "llerror.h" @@ -102,19 +103,95 @@ std::string LL::WorkQueueBase::makeName(const std::string& name) return STRINGIZE("WorkQueue" << num); } +namespace +{ +#if LL_WINDOWS + + static const U32 STATUS_MSC_EXCEPTION = 0xE06D7363; // compiler specific + + U32 exception_filter(U32 code, struct _EXCEPTION_POINTERS* exception_infop) + { + if (LLApp::instance()->reportCrashToBugsplat((void*)exception_infop)) + { + // Handled + return EXCEPTION_CONTINUE_SEARCH; + } + else if (code == STATUS_MSC_EXCEPTION) + { + // C++ exception, go on + return EXCEPTION_CONTINUE_SEARCH; + } + else + { + // handle it, convert to std::exception + return EXCEPTION_EXECUTE_HANDLER; + } + + return EXCEPTION_CONTINUE_SEARCH; + } + + void cpphandle(const LL::WorkQueueBase::Work& work) + { + // SE and C++ can not coexists, thus two handlers + try + { + work(); + } + catch (const LLContinueError&) + { + // Any uncaught exception derived from LLContinueError will be caught + // here and logged. This coroutine will terminate but the rest of the + // viewer will carry on. + LOG_UNHANDLED_EXCEPTION(STRINGIZE("LLContinue in work queue")); + } + } + + void sehandle(const LL::WorkQueueBase::Work& work) + { + __try + { + // handle stop and continue exceptions first + cpphandle(work); + } + __except (exception_filter(GetExceptionCode(), GetExceptionInformation())) + { + // convert to C++ styled exception + char integer_string[512]; + sprintf(integer_string, "SEH, code: %lu\n", GetExceptionCode()); + throw std::exception(integer_string); + } + } +#endif // LL_WINDOWS +} // anonymous namespace + void LL::WorkQueueBase::callWork(const Work& work) { LL_PROFILE_ZONE_SCOPED_CATEGORY_THREAD; + +#ifdef LL_WINDOWS + // can not use __try directly, toplevel requires unwinding, thus use of a wrapper + sehandle(work); +#else // LL_WINDOWS try { work(); } - catch (...) + catch (LLContinueError&) { - // No matter what goes wrong with any individual work item, the worker - // thread must go on! Log our own instance name with the exception. LOG_UNHANDLED_EXCEPTION(getKey()); } + catch (...) + { + // Stash any other kind of uncaught exception to be rethrown by main thread. + LL_WARNS("LLCoros") << "Capturing uncaught exception in WorkQueueBase " + << getKey() << LL_ENDL; + + LL::WorkQueue::ptr_t main_queue = LL::WorkQueue::getInstance("mainloop"); + main_queue->post( + // Bind the current exception rethrow it in main loop. + [exc = std::current_exception()]() { std::rethrow_exception(exc); }); + } +#endif // else LL_WINDOWS } void LL::WorkQueueBase::error(const std::string& msg) -- cgit v1.3 From e671cb54fee9cb7b7da39953eb96eabc9151f687 Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Sat, 29 Mar 2025 22:40:59 +0200 Subject: #3795 Crash at getVolatileAPRPool() --- indra/llcommon/workqueue.cpp | 4 ++-- indra/llui/llcommandmanager.cpp | 2 ++ indra/llui/llnotifications.cpp | 18 +++++++++++++++++- indra/llui/llnotifications.h | 2 +- indra/newview/llappviewer.cpp | 27 ++++++++++++++++++++------- indra/newview/llviewerwindow.cpp | 2 +- 6 files changed, 43 insertions(+), 12 deletions(-) (limited to 'indra/llcommon/workqueue.cpp') diff --git a/indra/llcommon/workqueue.cpp b/indra/llcommon/workqueue.cpp index dace95aaf2..c8ece616b2 100644 --- a/indra/llcommon/workqueue.cpp +++ b/indra/llcommon/workqueue.cpp @@ -183,12 +183,12 @@ void LL::WorkQueueBase::callWork(const Work& work) catch (...) { // Stash any other kind of uncaught exception to be rethrown by main thread. - LL_WARNS("LLCoros") << "Capturing uncaught exception in WorkQueueBase " + LL_WARNS("LLCoros") << "Capturing and rethrowing uncaught exception in WorkQueueBase " << getKey() << LL_ENDL; LL::WorkQueue::ptr_t main_queue = LL::WorkQueue::getInstance("mainloop"); main_queue->post( - // Bind the current exception rethrow it in main loop. + // Bind the current exception, rethrow it in main loop. [exc = std::current_exception()]() { std::rethrow_exception(exc); }); } #endif // else LL_WINDOWS diff --git a/indra/llui/llcommandmanager.cpp b/indra/llui/llcommandmanager.cpp index 03717da80b..b10ec51f18 100644 --- a/indra/llui/llcommandmanager.cpp +++ b/indra/llui/llcommandmanager.cpp @@ -170,12 +170,14 @@ bool LLCommandManager::load() if (!parser.readXUI(commands_file, commandsParams)) { + LLError::LLUserWarningMsg::showMissingFiles(); LL_ERRS() << "Unable to load xml file: " << commands_file << LL_ENDL; return false; } if (!commandsParams.validateBlock()) { + LLError::LLUserWarningMsg::showMissingFiles(); LL_ERRS() << "Invalid commands file: " << commands_file << LL_ENDL; return false; } diff --git a/indra/llui/llnotifications.cpp b/indra/llui/llnotifications.cpp index 634d8f71a2..a05feab1d9 100644 --- a/indra/llui/llnotifications.cpp +++ b/indra/llui/llnotifications.cpp @@ -1256,10 +1256,26 @@ LLNotifications::LLNotifications() LLInstanceTracker::instanceCount(); } + +LLNotifications::~LLNotifications() +{ + // Clear explicitly, something in ~LLNotifications() crashes so narrowing down suspects + pHistoryChannel = nullptr; + pExpirationChannel = nullptr; + mGlobalStrings.clear(); + mTemplates.clear(); + mVisibilityRules.clear(); + mUniqueNotifications.clear(); + mListener = nullptr; +} + void LLNotifications::clear() { mDefaultChannels.clear(); - mTemplates.clear(); + // At this point mTemplates still gets used by lingering notifications + // to do responses (ex: group notice will call forceResponse()), but + // since network should be down and everything save, it's questionable + // whether it should stay that way } // The expiration channel gets all notifications that are cancelled diff --git a/indra/llui/llnotifications.h b/indra/llui/llnotifications.h index 46286457cf..138f1969d5 100644 --- a/indra/llui/llnotifications.h +++ b/indra/llui/llnotifications.h @@ -887,7 +887,7 @@ class LLNotifications : { LLSINGLETON(LLNotifications); LOG_CLASS(LLNotifications); - virtual ~LLNotifications() {} + virtual ~LLNotifications(); public: diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 0ffe675a7b..7653b5ee8e 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5216,15 +5216,28 @@ void LLAppViewer::sendLogoutRequest() gLogoutInProgress = true; if (!mSecondInstance) { - mLogoutMarkerFileName = gDirUtilp->getExpandedFilename(LL_PATH_LOGS,LOGOUT_MARKER_FILE_NAME); - - mLogoutMarkerFile.open(mLogoutMarkerFileName, LL_APR_WB); - if (mLogoutMarkerFile.getFileHandle()) + mLogoutMarkerFileName = gDirUtilp->getExpandedFilename(LL_PATH_LOGS, LOGOUT_MARKER_FILE_NAME); + try { - LL_INFOS("MarkerFile") << "Created logout marker file '"<< mLogoutMarkerFileName << "' " << LL_ENDL; - recordMarkerVersion(mLogoutMarkerFile); + if (!mLogoutMarkerFile.getFileHandle()) + { + mLogoutMarkerFile.open(mLogoutMarkerFileName, LL_APR_WB); + if (mLogoutMarkerFile.getFileHandle()) + { + LL_INFOS("MarkerFile") << "Created logout marker file '" << mLogoutMarkerFileName << "' " << LL_ENDL; + recordMarkerVersion(mLogoutMarkerFile); + } + else + { + LL_WARNS("MarkerFile") << "Cannot create logout marker file " << mLogoutMarkerFileName << LL_ENDL; + } + } + else + { + LL_WARNS("MarkerFile") << "Atempted to reopen file '" << mLogoutMarkerFileName << "' " << LL_ENDL; + } } - else + catch (...) { LL_WARNS("MarkerFile") << "Cannot create logout marker file " << mLogoutMarkerFileName << LL_ENDL; } diff --git a/indra/newview/llviewerwindow.cpp b/indra/newview/llviewerwindow.cpp index fe5a0667a5..93ff175967 100644 --- a/indra/newview/llviewerwindow.cpp +++ b/indra/newview/llviewerwindow.cpp @@ -1457,7 +1457,7 @@ void LLViewerWindow::handleMouseLeave(LLWindow *window) bool LLViewerWindow::handleCloseRequest(LLWindow *window) { - if (!LLApp::isExiting()) + if (!LLApp::isExiting() && !LLApp::isStopped()) { // User has indicated they want to close, but we may need to ask // about modified documents. -- cgit v1.3 From 9117a37c3d1bd9741e599b60fdf000f9f174f72a Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Thu, 31 Oct 2024 02:30:54 +0200 Subject: Give control over thread queue's automatic shutdown --- indra/llcommon/threadpool.h | 2 +- indra/llcommon/workqueue.cpp | 15 +++++++++------ indra/llcommon/workqueue.h | 6 +++--- 3 files changed, 13 insertions(+), 10 deletions(-) (limited to 'indra/llcommon/workqueue.cpp') diff --git a/indra/llcommon/threadpool.h b/indra/llcommon/threadpool.h index edca52080d..ac4f415f3e 100644 --- a/indra/llcommon/threadpool.h +++ b/indra/llcommon/threadpool.h @@ -122,7 +122,7 @@ namespace LL size_t threads=1, size_t capacity=1024*1024, bool auto_shutdown = true): - ThreadPoolBase(name, threads, new queue_t(name, capacity), auto_shutdown) + ThreadPoolBase(name, threads, new queue_t(name, capacity, false), auto_shutdown) {} ~ThreadPoolUsing() override {} diff --git a/indra/llcommon/workqueue.cpp b/indra/llcommon/workqueue.cpp index 6066e74fb5..a2b056647c 100644 --- a/indra/llcommon/workqueue.cpp +++ b/indra/llcommon/workqueue.cpp @@ -29,12 +29,15 @@ using Lock = LLCoros::LockType; /***************************************************************************** * WorkQueueBase *****************************************************************************/ -LL::WorkQueueBase::WorkQueueBase(const std::string& name): - super(makeName(name)) +LL::WorkQueueBase::WorkQueueBase(const std::string& name, bool auto_shutdown) + : super(makeName(name)) +{ + if (auto_shutdown) { // TODO: register for "LLApp" events so we can implicitly close() on // viewer shutdown. } +} void LL::WorkQueueBase::runUntilClose() { @@ -135,8 +138,8 @@ void LL::WorkQueueBase::checkCoroutine(const std::string& method) /***************************************************************************** * WorkQueue *****************************************************************************/ -LL::WorkQueue::WorkQueue(const std::string& name, size_t capacity): - super(name), +LL::WorkQueue::WorkQueue(const std::string& name, size_t capacity, bool auto_shutdown): + super(name, auto_shutdown), mQueue(capacity) { } @@ -184,8 +187,8 @@ bool LL::WorkQueue::tryPop_(Work& work) /***************************************************************************** * WorkSchedule *****************************************************************************/ -LL::WorkSchedule::WorkSchedule(const std::string& name, size_t capacity): - super(name), +LL::WorkSchedule::WorkSchedule(const std::string& name, size_t capacity, bool auto_shutdown): + super(name, auto_shutdown), mQueue(capacity) { } diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h index 9d7bbfbf7a..d424ca6e1c 100644 --- a/indra/llcommon/workqueue.h +++ b/indra/llcommon/workqueue.h @@ -51,7 +51,7 @@ namespace LL * You may omit the WorkQueueBase name, in which case a unique name is * synthesized; for practical purposes that makes it anonymous. */ - WorkQueueBase(const std::string& name); + WorkQueueBase(const std::string& name, bool auto_shutdown); /** * Since the point of WorkQueue is to pass work to some other worker @@ -212,7 +212,7 @@ namespace LL * You may omit the WorkQueue name, in which case a unique name is * synthesized; for practical purposes that makes it anonymous. */ - WorkQueue(const std::string& name = std::string(), size_t capacity=1024); + WorkQueue(const std::string& name = std::string(), size_t capacity=1024, bool auto_shutdown = true); /** * Since the point of WorkQueue is to pass work to some other worker @@ -282,7 +282,7 @@ namespace LL * You may omit the WorkSchedule name, in which case a unique name is * synthesized; for practical purposes that makes it anonymous. */ - WorkSchedule(const std::string& name = std::string(), size_t capacity=1024); + WorkSchedule(const std::string& name = std::string(), size_t capacity=1024, bool auto_shutdown = true); /** * Since the point of WorkSchedule is to pass work to some other worker -- cgit v1.3 From ec6c988bbbc59aed218d3629bf0c13192f6b726c Mon Sep 17 00:00:00 2001 From: Andrey Lihatskiy Date: Wed, 23 Apr 2025 01:27:32 +0300 Subject: #3918 Implement automatic shutdown for WorkQueueBase --- indra/llcommon/workqueue.cpp | 31 ++++++++++++++++++++++++++++--- indra/llcommon/workqueue.h | 5 +++++ 2 files changed, 33 insertions(+), 3 deletions(-) (limited to 'indra/llcommon/workqueue.cpp') diff --git a/indra/llcommon/workqueue.cpp b/indra/llcommon/workqueue.cpp index 323903e59c..0eb20323ad 100644 --- a/indra/llcommon/workqueue.cpp +++ b/indra/llcommon/workqueue.cpp @@ -21,6 +21,7 @@ #include "llcoros.h" #include LLCOROS_MUTEX_HEADER #include "llerror.h" +#include "llevents.h" #include "llexception.h" #include "stringize.h" @@ -34,10 +35,34 @@ LL::WorkQueueBase::WorkQueueBase(const std::string& name, bool auto_shutdown) : super(makeName(name)) { if (auto_shutdown) -{ - // TODO: register for "LLApp" events so we can implicitly close() on - // viewer shutdown. + { + // Register for "LLApp" events so we can implicitly close() on viewer shutdown + std::string listener_name = "WorkQueue:" + getKey(); + LLEventPumps::instance().obtain("LLApp").listen( + listener_name, + [this](const LLSD& stat) + { + std::string status(stat["status"]); + if (status != "running") + { + // Viewer is shutting down, close this queue + LL_DEBUGS("WorkQueue") << getKey() << " closing on app shutdown" << LL_ENDL; + close(); + } + return false; + }); + + // Store the listener name so we can unregister in the destructor + mListenerName = listener_name; + } } + +LL::WorkQueueBase::~WorkQueueBase() +{ + if (!mListenerName.empty() && !LLEventPumps::wasDeleted()) + { + LLEventPumps::instance().obtain("LLApp").stopListening(mListenerName); + } } void LL::WorkQueueBase::runUntilClose() diff --git a/indra/llcommon/workqueue.h b/indra/llcommon/workqueue.h index d424ca6e1c..735ad38a26 100644 --- a/indra/llcommon/workqueue.h +++ b/indra/llcommon/workqueue.h @@ -53,6 +53,8 @@ namespace LL */ WorkQueueBase(const std::string& name, bool auto_shutdown); + virtual ~WorkQueueBase(); + /** * Since the point of WorkQueue is to pass work to some other worker * thread(s) asynchronously, it's important that it continue to exist @@ -197,6 +199,9 @@ namespace LL private: virtual Work pop_() = 0; virtual bool tryPop_(Work&) = 0; + + // Name used for the LLApp event listener (empty if not registered) + std::string mListenerName; }; /***************************************************************************** -- cgit v1.3 From 4b69fe396fc0bb783c7db203fd327c16a30f6c5e Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Sat, 19 Jul 2025 10:12:45 +0300 Subject: #4320 MacOS crash handling --- indra/llcommon/workqueue.cpp | 24 ++++++++++++++++-------- indra/newview/llappviewer.cpp | 3 ++- 2 files changed, 18 insertions(+), 9 deletions(-) (limited to 'indra/llcommon/workqueue.cpp') diff --git a/indra/llcommon/workqueue.cpp b/indra/llcommon/workqueue.cpp index c8ece616b2..ea4feec58c 100644 --- a/indra/llcommon/workqueue.cpp +++ b/indra/llcommon/workqueue.cpp @@ -182,14 +182,22 @@ void LL::WorkQueueBase::callWork(const Work& work) } catch (...) { - // Stash any other kind of uncaught exception to be rethrown by main thread. - LL_WARNS("LLCoros") << "Capturing and rethrowing uncaught exception in WorkQueueBase " - << getKey() << LL_ENDL; - - LL::WorkQueue::ptr_t main_queue = LL::WorkQueue::getInstance("mainloop"); - main_queue->post( - // Bind the current exception, rethrow it in main loop. - [exc = std::current_exception()]() { std::rethrow_exception(exc); }); + if (getKey() != "mainloop") + { + // Stash any other kind of uncaught exception to be rethrown by main thread. + LL_WARNS("LLCoros") << "Capturing and rethrowing uncaught exception in WorkQueueBase " + << getKey() << LL_ENDL; + + LL::WorkQueue::ptr_t main_queue = LL::WorkQueue::getInstance("mainloop"); + main_queue->post( + // Bind the current exception, rethrow it in main loop. + [exc = std::current_exception()]() { std::rethrow_exception(exc); }); + } + else + { + // let main loop crash + throw; + } } #endif // else LL_WINDOWS } diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index f4b2fdfdf7..35fdc18839 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5696,7 +5696,8 @@ void LLAppViewer::forceExceptionThreadCrash() void run() { - throw std::exception(); + const std::string exception_text = "This is a deliberate exception in a thread"; + throw std::runtime_error(exception_text); } }; -- cgit v1.3