From 85cba58ad473ed28efda7f645af20d56229e8637 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 22 Aug 2014 18:04:27 -0400 Subject: Add an HTTP policy class for inventory operations using four (4) connections. Convert background and foreground fetches, both items and folders/inventory and library, to use new HTTP. Non-fetch inventory operations continue to use LLHTTPClient (at least for now). Error handling and retry on fetches wasn't 100% previously and that's still the case. I'll rip through this again to clean that up. Cleaned up logging in much of the inventory code with consistent labels on logging events and correct macros (removed deprecation warnings). This started as an attempt to get libcurl to do pipelining on POSTs and PUTs. Discovered that this is going to be very difficult to support in general in libcurl. May look at that again in the future. --- indra/newview/llinventorymodelbackgroundfetch.cpp | 740 +++++++++++++--------- 1 file changed, 450 insertions(+), 290 deletions(-) (limited to 'indra/newview/llinventorymodelbackgroundfetch.cpp') diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 2de37b0790..443c54df42 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2002&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -37,31 +37,113 @@ #include "llviewermessage.h" #include "llviewerregion.h" #include "llviewerwindow.h" +#include "llhttpconstants.h" +#include "bufferarray.h" +#include "bufferstream.h" +#include "llsdserialize.h" + +namespace +{ + +// Http request handler class for single inventory item requests. +// +// We'll use a handler-per-request pattern here rather than +// a shared handler. Mainly convenient as this was converted +// from a Responder class model. +// +class BGItemHttpHandler : public LLInventoryModel::FetchItemHttpHandler +{ + LOG_CLASS(BGItemHttpHandler); + +public: + BGItemHttpHandler(const LLSD & request_sd) + : LLInventoryModel::FetchItemHttpHandler(request_sd) + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); + } + + virtual ~BGItemHttpHandler() + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); + } + +protected: + BGItemHttpHandler(const BGItemHttpHandler &); // Not defined + void operator=(const BGItemHttpHandler &); // Not defined +}; + + +// Http request handler class for folders. +class BGFolderHttpHandler : public LLCore::HttpHandler +{ + LOG_CLASS(BGFolderHttpHandler); + +public: + BGFolderHttpHandler(const LLSD & request_sd, const uuid_vec_t & recursive_cats) + : LLCore::HttpHandler(), + mRequestSD(request_sd), + mRecursiveCatUUIDs(recursive_cats) + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); + } + + virtual ~BGFolderHttpHandler() + { + LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); + } + +protected: + BGFolderHttpHandler(const BGFolderHttpHandler &); // Not defined + void operator=(const BGFolderHttpHandler &); // Not defined + +public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + + bool getIsRecursive(const LLUUID & cat_id) const; + +private: + void processData(LLSD & body, LLCore::HttpResponse * response); + void processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response); + void processFailure(const char * const reason, LLCore::HttpResponse * response); + +private: + LLSD mRequestSD; + const uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive +}; + -const F32 MAX_TIME_FOR_SINGLE_FETCH = 10.f; const S32 MAX_FETCH_RETRIES = 10; -LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch() : +const char * const LOG_INV("Inventory"); + +std::string dumpResponse() +{ + return std::string("ADD SOMETHING MEANINGFUL HERE"); +} + + +} // end of namespace anonymous + + +LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch(): mBackgroundFetchActive(FALSE), mFolderFetchActive(false), + mFetchCount(0), mAllFoldersFetched(FALSE), mRecursiveInventoryFetchStarted(FALSE), mRecursiveLibraryFetchStarted(FALSE), mNumFetchRetries(0), mMinTimeBetweenFetches(0.3f), mMaxTimeBetweenFetches(10.f), - mTimelyFetchPending(FALSE), - mFetchCount(0) -{ -} + mTimelyFetchPending(FALSE) +{} LLInventoryModelBackgroundFetch::~LLInventoryModelBackgroundFetch() -{ -} +{} bool LLInventoryModelBackgroundFetch::isBulkFetchProcessingComplete() const { - return mFetchQueue.empty() && mFetchCount<=0; + return mFetchQueue.empty() && mFetchCount <= 0; } bool LLInventoryModelBackgroundFetch::libraryFetchStarted() const @@ -91,7 +173,7 @@ bool LLInventoryModelBackgroundFetch::inventoryFetchCompleted() const bool LLInventoryModelBackgroundFetch::inventoryFetchInProgress() const { - return inventoryFetchStarted() && !inventoryFetchCompleted(); + return inventoryFetchStarted() && ! inventoryFetchCompleted(); } bool LLInventoryModelBackgroundFetch::isEverythingFetched() const @@ -104,24 +186,36 @@ BOOL LLInventoryModelBackgroundFetch::folderFetchActive() const return mFolderFetchActive; } +void LLInventoryModelBackgroundFetch::addRequestAtFront(const LLUUID & id, BOOL recursive, bool is_category) +{ + mFetchQueue.push_front(FetchQueueInfo(id, recursive, is_category)); +} + +void LLInventoryModelBackgroundFetch::addRequestAtBack(const LLUUID & id, BOOL recursive, bool is_category) +{ + mFetchQueue.push_back(FetchQueueInfo(id, recursive, is_category)); +} + void LLInventoryModelBackgroundFetch::start(const LLUUID& id, BOOL recursive) { - LLViewerInventoryCategory* cat = gInventory.getCategory(id); - if (cat || (id.isNull() && !isEverythingFetched())) - { // it's a folder, do a bulk fetch - LL_DEBUGS("InventoryFetch") << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL; + LLViewerInventoryCategory * cat(gInventory.getCategory(id)); + + if (cat || (id.isNull() && ! isEverythingFetched())) + { + // it's a folder, do a bulk fetch + LL_DEBUGS(LOG_INV) << "Start fetching category: " << id << ", recursive: " << recursive << LL_ENDL; mBackgroundFetchActive = TRUE; mFolderFetchActive = true; if (id.isNull()) { - if (!mRecursiveInventoryFetchStarted) + if (! mRecursiveInventoryFetchStarted) { mRecursiveInventoryFetchStarted |= recursive; mFetchQueue.push_back(FetchQueueInfo(gInventory.getRootFolderID(), recursive)); gIdleCallbacks.addFunction(&LLInventoryModelBackgroundFetch::backgroundFetchCB, NULL); } - if (!mRecursiveLibraryFetchStarted) + if (! mRecursiveLibraryFetchStarted) { mRecursiveLibraryFetchStarted |= recursive; mFetchQueue.push_back(FetchQueueInfo(gInventory.getLibraryRootFolderID(), recursive)); @@ -146,9 +240,9 @@ void LLInventoryModelBackgroundFetch::start(const LLUUID& id, BOOL recursive) } } } - else if (LLViewerInventoryItem* itemp = gInventory.getItem(id)) + else if (LLViewerInventoryItem * itemp = gInventory.getItem(id)) { - if (!itemp->mIsComplete && (mFetchQueue.empty() || mFetchQueue.front().mUUID != id)) + if (! itemp->mIsComplete && (mFetchQueue.empty() || mFetchQueue.front().mUUID != id)) { mBackgroundFetchActive = TRUE; @@ -172,7 +266,7 @@ void LLInventoryModelBackgroundFetch::setAllFoldersFetched() mRecursiveLibraryFetchStarted) { mAllFoldersFetched = TRUE; - //LL_INFOS() << "All folders fetched, validating" << LL_ENDL; + //LL_INFOS(LOG_INV) << "All folders fetched, validating" << LL_ENDL; //gInventory.validate(); } mFolderFetchActive = false; @@ -203,7 +297,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // No more categories to fetch, stop fetch process. if (mFetchQueue.empty()) { - LL_INFOS() << "Inventory fetch completed" << LL_ENDL; + LL_INFOS(LOG_INV) << "Inventory fetch completed" << LL_ENDL; setAllFoldersFetched(); mBackgroundFetchActive = false; @@ -219,7 +313,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // Double timeouts on failure. mMinTimeBetweenFetches = llmin(mMinTimeBetweenFetches * 2.f, 10.f); mMaxTimeBetweenFetches = llmin(mMaxTimeBetweenFetches * 2.f, 120.f); - LL_DEBUGS() << "Inventory fetch times grown to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Inventory fetch times grown to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; // fetch is no longer considered "timely" although we will wait for full time-out. mTimelyFetchPending = FALSE; } @@ -231,7 +325,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() break; } - if(gDisconnected) + if (gDisconnected) { // Just bail if we are disconnected. break; @@ -292,7 +386,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // Shrink timeouts based on success. mMinTimeBetweenFetches = llmax(mMinTimeBetweenFetches * 0.8f, 0.3f); mMaxTimeBetweenFetches = llmax(mMaxTimeBetweenFetches * 0.8f, 10.f); - LL_DEBUGS() << "Inventory fetch times shrunk to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; + LL_DEBUGS(LOG_INV) << "Inventory fetch times shrunk to (" << mMinTimeBetweenFetches << ", " << mMaxTimeBetweenFetches << ")" << LL_ENDL; } mTimelyFetchPending = FALSE; @@ -355,257 +449,53 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() } } -void LLInventoryModelBackgroundFetch::incrFetchCount(S16 fetching) +void LLInventoryModelBackgroundFetch::incrFetchCount(S32 fetching) { mFetchCount += fetching; if (mFetchCount < 0) { + LL_WARNS_ONCE(LOG_INV) << "Inventory fetch count fell below zero (0)." << LL_ENDL; mFetchCount = 0; } } -class LLInventoryModelFetchItemResponder : public LLInventoryModel::fetchInventoryResponder -{ - LOG_CLASS(LLInventoryModelFetchItemResponder); -public: - LLInventoryModelFetchItemResponder(const LLSD& request_sd) : - LLInventoryModel::fetchInventoryResponder(request_sd) - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); - } -private: - /* virtual */ void httpCompleted() - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); - LLInventoryModel::fetchInventoryResponder::httpCompleted(); - } -}; - -class LLInventoryModelFetchDescendentsResponder: public LLHTTPClient::Responder -{ - LOG_CLASS(LLInventoryModelFetchDescendentsResponder); -public: - LLInventoryModelFetchDescendentsResponder(const LLSD& request_sd, uuid_vec_t recursive_cats) : - mRequestSD(request_sd), - mRecursiveCatUUIDs(recursive_cats) - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(1); - } - //LLInventoryModelFetchDescendentsResponder() {}; -private: - /* virtual */ void httpCompleted() - { - LLInventoryModelBackgroundFetch::instance().incrFetchCount(-1); - LLHTTPClient::Responder::httpCompleted(); - } - /* virtual */ void httpSuccess(); - /* virtual */ void httpFailure(); -protected: - BOOL getIsRecursive(const LLUUID& cat_id) const; -private: - LLSD mRequestSD; - uuid_vec_t mRecursiveCatUUIDs; // hack for storing away which cat fetches are recursive -}; - -// If we get back a normal response, handle it here. -void LLInventoryModelFetchDescendentsResponder::httpSuccess() +// Bundle up a bunch of requests to send all at once. +void LLInventoryModelBackgroundFetch::bulkFetch() { - const LLSD& content = getContent(); - if (!content.isMap()) + //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped. + //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was + //sent. If it exceeds our retry time, go ahead and fire off another batch. + LLViewerRegion * region(gAgent.getRegion()); + if (! region || gDisconnected) { - failureResult(HTTP_INTERNAL_ERROR, "Malformed response contents", content); return; } - LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - if (content.has("folders")) - { - - for(LLSD::array_const_iterator folder_it = content["folders"].beginArray(); - folder_it != content["folders"].endArray(); - ++folder_it) - { - LLSD folder_sd = *folder_it; - - - //LLUUID agent_id = folder_sd["agent_id"]; - - //if(agent_id != gAgent.getID()) //This should never happen. - //{ - // LL_WARNS() << "Got a UpdateInventoryItem for the wrong agent." - // << LL_ENDL; - // break; - //} - - LLUUID parent_id = folder_sd["folder_id"]; - LLUUID owner_id = folder_sd["owner_id"]; - S32 version = (S32)folder_sd["version"].asInteger(); - S32 descendents = (S32)folder_sd["descendents"].asInteger(); - LLPointer tcategory = new LLViewerInventoryCategory(owner_id); - if (parent_id.isNull()) - { - LLPointer titem = new LLViewerInventoryItem; - for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); - item_it != folder_sd["items"].endArray(); - ++item_it) - { - const LLUUID lost_uuid = gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND); - if (lost_uuid.notNull()) - { - LLSD item = *item_it; - titem->unpackMessage(item); - - LLInventoryModel::update_list_t update; - LLInventoryModel::LLCategoryUpdate new_folder(lost_uuid, 1); - update.push_back(new_folder); - gInventory.accountForUpdate(update); - - titem->setParent(lost_uuid); - titem->updateParentOnServer(FALSE); - gInventory.updateItem(titem); - gInventory.notifyObservers(); - - } - } - } - - LLViewerInventoryCategory* pcat = gInventory.getCategory(parent_id); - if (!pcat) - { - continue; - } - - for(LLSD::array_const_iterator category_it = folder_sd["categories"].beginArray(); - category_it != folder_sd["categories"].endArray(); - ++category_it) - { - LLSD category = *category_it; - tcategory->fromLLSD(category); - - const BOOL recursive = getIsRecursive(tcategory->getUUID()); - - if (recursive) - { - fetcher->mFetchQueue.push_back(LLInventoryModelBackgroundFetch::FetchQueueInfo(tcategory->getUUID(), recursive)); - } - else if ( !gInventory.isCategoryComplete(tcategory->getUUID()) ) - { - gInventory.updateCategory(tcategory); - } - - } - LLPointer titem = new LLViewerInventoryItem; - for(LLSD::array_const_iterator item_it = folder_sd["items"].beginArray(); - item_it != folder_sd["items"].endArray(); - ++item_it) - { - LLSD item = *item_it; - titem->unpackMessage(item); - - gInventory.updateItem(titem); - } - - // Set version and descendentcount according to message. - LLViewerInventoryCategory* cat = gInventory.getCategory(parent_id); - if(cat) - { - cat->setVersion(version); - cat->setDescendentCount(descendents); - cat->determineFolderType(); - } - - } - } - - if (content.has("bad_folders")) - { - for(LLSD::array_const_iterator folder_it = content["bad_folders"].beginArray(); - folder_it != content["bad_folders"].endArray(); - ++folder_it) - { - // *TODO: Stop copying data - LLSD folder_sd = *folder_it; - - // These folders failed on the dataserver. We probably don't want to retry them. - LL_WARNS() << "Folder " << folder_sd["folder_id"].asString() - << "Error: " << folder_sd["error"].asString() << LL_ENDL; - } - } + static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections + static const F32 new_min_time(0.5f); // *HACK: Clean this up when old code goes away entirely. + static const U32 max_batch_size(10); - if (fetcher->isBulkFetchProcessingComplete()) + if (mMinTimeBetweenFetches < new_min_time) { - LL_INFOS() << "Inventory fetch completed" << LL_ENDL; - fetcher->setAllFoldersFetched(); + mMinTimeBetweenFetches = new_min_time; // *HACK: See above. } - - gInventory.notifyObservers(); -} - -// If we get back an error (not found, etc...), handle it here. -void LLInventoryModelFetchDescendentsResponder::httpFailure() -{ - LL_WARNS() << dumpResponse() << LL_ENDL; - LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - - LL_INFOS() << dumpResponse() << LL_ENDL; - - fetcher->incrFetchCount(-1); - if (getStatus()==HTTP_INTERNAL_ERROR) // timed out or curl failure - { - for(LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); - folder_it != mRequestSD["folders"].endArray(); - ++folder_it) - { - LLSD folder_sd = *folder_it; - LLUUID folder_id = folder_sd["folder_id"]; - const BOOL recursive = getIsRecursive(folder_id); - fetcher->mFetchQueue.push_front(LLInventoryModelBackgroundFetch::FetchQueueInfo(folder_id, recursive)); - } - } - else + if (mFetchCount) { - if (fetcher->isBulkFetchProcessingComplete()) - { - fetcher->setAllFoldersFetched(); - } - } - gInventory.notifyObservers(); -} - -BOOL LLInventoryModelFetchDescendentsResponder::getIsRecursive(const LLUUID& cat_id) const -{ - return (std::find(mRecursiveCatUUIDs.begin(),mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end()); -} -// Bundle up a bunch of requests to send all at once. -// static -void LLInventoryModelBackgroundFetch::bulkFetch() -{ - //Background fetch is called from gIdleCallbacks in a loop until background fetch is stopped. - //If there are items in mFetchQueue, we want to check the time since the last bulkFetch was - //sent. If it exceeds our retry time, go ahead and fire off another batch. - LLViewerRegion* region = gAgent.getRegion(); - if (!region) return; - - S16 max_concurrent_fetches=8; - F32 new_min_time = 0.5f; //HACK! Clean this up when old code goes away entirely. - if (mMinTimeBetweenFetches < new_min_time) - { - mMinTimeBetweenFetches=new_min_time; //HACK! See above. + // Process completed HTTP requests + gInventory.handleResponses(false); } - if (gDisconnected || - (mFetchCount > max_concurrent_fetches) || + if ((mFetchCount > max_concurrent_fetches) || (mFetchTimer.getElapsedTimeF32() < mMinTimeBetweenFetches)) { - return; // just bail if we are disconnected + return; } - U32 item_count=0; - U32 folder_count=0; - U32 max_batch_size=5; + U32 item_count(0); + U32 folder_count(0); - U32 sort_order = gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1; + const U32 sort_order(gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1); uuid_vec_t recursive_cats; @@ -614,27 +504,27 @@ void LLInventoryModelBackgroundFetch::bulkFetch() LLSD item_request_body; LLSD item_request_body_lib; - while (!mFetchQueue.empty() + while (! mFetchQueue.empty() && (item_count + folder_count) < max_batch_size) { - const FetchQueueInfo& fetch_info = mFetchQueue.front(); + const FetchQueueInfo & fetch_info(mFetchQueue.front()); if (fetch_info.mIsCategory) { - const LLUUID &cat_id = fetch_info.mUUID; + const LLUUID & cat_id(fetch_info.mUUID); if (cat_id.isNull()) //DEV-17797 { LLSD folder_sd; folder_sd["folder_id"] = LLUUID::null.asString(); folder_sd["owner_id"] = gAgent.getID(); - folder_sd["sort_order"] = (LLSD::Integer)sort_order; - folder_sd["fetch_folders"] = (LLSD::Boolean)FALSE; - folder_sd["fetch_items"] = (LLSD::Boolean)TRUE; + folder_sd["sort_order"] = LLSD::Integer(sort_order); + folder_sd["fetch_folders"] = LLSD::Boolean(FALSE); + folder_sd["fetch_items"] = LLSD::Boolean(TRUE); folder_request_body["folders"].append(folder_sd); folder_count++; } else { - const LLViewerInventoryCategory* cat = gInventory.getCategory(cat_id); + const LLViewerInventoryCategory * cat(gInventory.getCategory(cat_id)); if (cat) { @@ -643,21 +533,26 @@ void LLInventoryModelBackgroundFetch::bulkFetch() LLSD folder_sd; folder_sd["folder_id"] = cat->getUUID(); folder_sd["owner_id"] = cat->getOwnerID(); - folder_sd["sort_order"] = (LLSD::Integer)sort_order; - folder_sd["fetch_folders"] = TRUE; //(LLSD::Boolean)sFullFetchStarted; - folder_sd["fetch_items"] = (LLSD::Boolean)TRUE; + folder_sd["sort_order"] = LLSD::Integer(sort_order); + folder_sd["fetch_folders"] = LLSD::Boolean(TRUE); //(LLSD::Boolean)sFullFetchStarted; + folder_sd["fetch_items"] = LLSD::Boolean(TRUE); if (ALEXANDRIA_LINDEN_ID == cat->getOwnerID()) + { folder_request_body_lib["folders"].append(folder_sd); + } else + { folder_request_body["folders"].append(folder_sd); + } folder_count++; } + // May already have this folder, but append child folders to list. if (fetch_info.mRecursive) { - LLInventoryModel::cat_array_t* categories; - LLInventoryModel::item_array_t* items; + LLInventoryModel::cat_array_t * categories(NULL); + LLInventoryModel::item_array_t * items(NULL); gInventory.getDirectDescendentsOf(cat->getUUID(), categories, items); for (LLInventoryModel::cat_array_t::const_iterator it = categories->begin(); it != categories->end(); @@ -669,11 +564,14 @@ void LLInventoryModelBackgroundFetch::bulkFetch() } } if (fetch_info.mRecursive) + { recursive_cats.push_back(cat_id); + } } else { - LLViewerInventoryItem* itemp = gInventory.getItem(fetch_info.mUUID); + LLViewerInventoryItem * itemp(gInventory.getItem(fetch_info.mUUID)); + if (itemp) { LLSD item_sd; @@ -694,72 +592,80 @@ void LLInventoryModelBackgroundFetch::bulkFetch() mFetchQueue.pop_front(); } - + + // Issue HTTP POST requests to fetch folders and items + if (item_count + folder_count > 0) { if (folder_count) { - std::string url = region->getCapability("FetchInventoryDescendents2"); - if ( !url.empty() ) + if (folder_request_body["folders"].size()) { - if (folder_request_body["folders"].size()) + const std::string url(region->getCapability("FetchInventoryDescendents2")); + + if (! url.empty()) { - LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body, recursive_cats); - LLHTTPClient::post(url, folder_request_body, fetcher, 300.0); + BGFolderHttpHandler * handler(new BGFolderHttpHandler(folder_request_body, recursive_cats)); + gInventory.requestPost(false, url, folder_request_body, handler, "Inventory Folder"); } - if (folder_request_body_lib["folders"].size()) - { - std::string url_lib = gAgent.getRegion()->getCapability("FetchLibDescendents2"); + } + + if (folder_request_body_lib["folders"].size()) + { + const std::string url(region->getCapability("FetchLibDescendents2")); - LLInventoryModelFetchDescendentsResponder *fetcher = new LLInventoryModelFetchDescendentsResponder(folder_request_body_lib, recursive_cats); - LLHTTPClient::post(url_lib, folder_request_body_lib, fetcher, 300.0); + if (! url.empty()) + { + BGFolderHttpHandler * handler(new BGFolderHttpHandler(folder_request_body_lib, recursive_cats)); + gInventory.requestPost(false, url, folder_request_body_lib, handler, "Library Folder"); } } - } + } // if (folder_count) + if (item_count) { - std::string url; - if (item_request_body.size()) { - url = region->getCapability("FetchInventory2"); - if (!url.empty()) + const std::string url(region->getCapability("FetchInventory2")); + + if (! url.empty()) { LLSD body; body["items"] = item_request_body; - - LLHTTPClient::post(url, body, new LLInventoryModelFetchItemResponder(body)); + BGItemHttpHandler * handler(new BGItemHttpHandler(body)); + gInventory.requestPost(false, url, body, handler, "Inventory Item"); } } if (item_request_body_lib.size()) { + const std::string url(region->getCapability("FetchLib2")); - url = region->getCapability("FetchLib2"); - if (!url.empty()) + if (! url.empty()) { LLSD body; body["items"] = item_request_body_lib; - - LLHTTPClient::post(url, body, new LLInventoryModelFetchItemResponder(body)); + BGItemHttpHandler * handler(new BGItemHttpHandler(body)); + gInventory.requestPost(false, url, body, handler, "Library Item"); } } - } + } // if (item_count) + mFetchTimer.reset(); } - else if (isBulkFetchProcessingComplete()) { setAllFoldersFetched(); } } -bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID& cat_id) const +bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LLUUID & cat_id) const { for (fetch_queue_t::const_iterator it = mFetchQueue.begin(); - it != mFetchQueue.end(); ++it) + it != mFetchQueue.end(); + ++it) { - const LLUUID& fetch_id = (*it).mUUID; + const LLUUID & fetch_id = (*it).mUUID; if (gInventory.isObjectDescendentOf(fetch_id, cat_id)) return false; } @@ -767,3 +673,257 @@ bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LL } +// Anonymous Namespace Definitions + +namespace +{ + +// ==== BGFolderHttpHandler ==== + +void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + LLCore::HttpStatus status(response->getStatus()); + if (! status) + { + processFailure(status, response); + } + else + { + LLCore::BufferArray * body(response->getBody()); + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; + processFailure("HTTP response missing expected body", response); + goto only_exit; + } + + LLCore::BufferArrayStream bas(body); + LLSD body_llsd; + S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas)); + if (LLSDParser::PARSE_FAILURE == parse_status) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response contained malformed LLSD", response); + goto only_exit; + } + + // Okay, process data if possible + processData(body_llsd, response); + } + +only_exit: + // Must delete on completion. + delete this; +} + + +void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) +{ + if (! content.isMap()) + { + processFailure("LLSD response not a map", response); + return; + } + + LLInventoryModelBackgroundFetch * fetcher(LLInventoryModelBackgroundFetch::getInstance()); + if (content.has("folders")) + { + LLSD folders(content["folders"]); + + for (LLSD::array_const_iterator folder_it = folders.beginArray(); + folder_it != folders.endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + + //LLUUID agent_id = folder_sd["agent_id"]; + + //if(agent_id != gAgent.getID()) //This should never happen. + //{ + // LL_WARNS(LOG_INV) << "Got a UpdateInventoryItem for the wrong agent." + // << LL_ENDL; + // break; + //} + + LLUUID parent_id(folder_sd["folder_id"].asUUID()); + LLUUID owner_id(folder_sd["owner_id"].asUUID()); + S32 version(folder_sd["version"].asInteger()); + S32 descendents(folder_sd["descendents"].asInteger()); + LLPointer tcategory = new LLViewerInventoryCategory(owner_id); + + if (parent_id.isNull()) + { + LLSD items(folder_sd["items"]); + LLPointer titem = new LLViewerInventoryItem; + + for (LLSD::array_const_iterator item_it = items.beginArray(); + item_it != items.endArray(); + ++item_it) + { + const LLUUID lost_uuid(gInventory.findCategoryUUIDForType(LLFolderType::FT_LOST_AND_FOUND)); + + if (lost_uuid.notNull()) + { + LLSD item(*item_it); + + titem->unpackMessage(item); + + LLInventoryModel::update_list_t update; + LLInventoryModel::LLCategoryUpdate new_folder(lost_uuid, 1); + update.push_back(new_folder); + gInventory.accountForUpdate(update); + + titem->setParent(lost_uuid); + titem->updateParentOnServer(FALSE); + gInventory.updateItem(titem); + gInventory.notifyObservers(); + } + } + } + + LLViewerInventoryCategory * pcat(gInventory.getCategory(parent_id)); + if (! pcat) + { + continue; + } + + LLSD categories(folder_sd["categories"]); + for (LLSD::array_const_iterator category_it = categories.beginArray(); + category_it != categories.endArray(); + ++category_it) + { + LLSD category(*category_it); + tcategory->fromLLSD(category); + + const bool recursive(getIsRecursive(tcategory->getUUID())); + if (recursive) + { + fetcher->addRequestAtBack(tcategory->getUUID(), recursive, true); + } + else if (! gInventory.isCategoryComplete(tcategory->getUUID())) + { + gInventory.updateCategory(tcategory); + } + } + + LLSD items(folder_sd["items"]); + LLPointer titem = new LLViewerInventoryItem; + for (LLSD::array_const_iterator item_it = items.beginArray(); + item_it != items.endArray(); + ++item_it) + { + LLSD item(*item_it); + titem->unpackMessage(item); + + gInventory.updateItem(titem); + } + + // Set version and descendentcount according to message. + LLViewerInventoryCategory * cat(gInventory.getCategory(parent_id)); + if (cat) + { + cat->setVersion(version); + cat->setDescendentCount(descendents); + cat->determineFolderType(); + } + } + } + + if (content.has("bad_folders")) + { + LLSD bad_folders(content["bad_folders"]); + for (LLSD::array_const_iterator folder_it = bad_folders.beginArray(); + folder_it != bad_folders.endArray(); + ++folder_it) + { + // *TODO: Stop copying data + LLSD folder_sd(*folder_it); + + // These folders failed on the dataserver. We probably don't want to retry them. + LL_WARNS(LOG_INV) << "Folder " << folder_sd["folder_id"].asString() + << "Error: " << folder_sd["error"].asString() << LL_ENDL; + } + } + + if (fetcher->isBulkFetchProcessingComplete()) + { + LL_INFOS(LOG_INV) << "Inventory fetch completed" << LL_ENDL; + fetcher->setAllFoldersFetched(); + } + + gInventory.notifyObservers(); +} + + +void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) +{ + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); + + LL_INFOS(LOG_INV) << dumpResponse() << LL_ENDL; + + // *FIX: Not the correct test here... + if (status == LLCore::HttpStatus(408)) // timed out or curl failure + { + for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); + folder_it != mRequestSD["folders"].endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + LLUUID folder_id(folder_sd["folder_id"]); + const BOOL recursive = getIsRecursive(folder_id); + fetcher->addRequestAtFront(folder_id, recursive, true); + } + } + else + { + if (fetcher->isBulkFetchProcessingComplete()) + { + fetcher->setAllFoldersFetched(); + } + } + gInventory.notifyObservers(); +} + + +void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) +{ + LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); + + LL_INFOS(LOG_INV) << dumpResponse() << LL_ENDL; + + if (true /* status == LLCore::HttpStatus(408)*/) // timed out or curl failure + { + for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); + folder_it != mRequestSD["folders"].endArray(); + ++folder_it) + { + LLSD folder_sd(*folder_it); + LLUUID folder_id(folder_sd["folder_id"]); + const BOOL recursive = getIsRecursive(folder_id); + fetcher->addRequestAtFront(folder_id, recursive, true); + } + } + else + { + if (fetcher->isBulkFetchProcessingComplete()) + { + fetcher->setAllFoldersFetched(); + } + } + gInventory.notifyObservers(); +} + + +bool BGFolderHttpHandler::getIsRecursive(const LLUUID & cat_id) const +{ + return std::find(mRecursiveCatUUIDs.begin(), mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end(); +} + + +// ==== BGItemHttpHandler ==== + +// Nothing to implement here. All ctor/dtor changes. + +} // end namespace anonymous -- cgit v1.3 From b64ef2ecd4af5265483527f2ef030554133ab137 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 22 Aug 2014 19:00:11 -0400 Subject: Fix ambiguous constructor due to LLSD access which broke *ix builds. --- indra/newview/llinventorymodelbackgroundfetch.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'indra/newview/llinventorymodelbackgroundfetch.cpp') diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 443c54df42..7b944edf45 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -870,7 +870,7 @@ void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::Http ++folder_it) { LLSD folder_sd(*folder_it); - LLUUID folder_id(folder_sd["folder_id"]); + LLUUID folder_id(folder_sd["folder_id"].asUUID()); const BOOL recursive = getIsRecursive(folder_id); fetcher->addRequestAtFront(folder_id, recursive, true); } @@ -900,7 +900,7 @@ void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::Http ++folder_it) { LLSD folder_sd(*folder_it); - LLUUID folder_id(folder_sd["folder_id"]); + LLUUID folder_id(folder_sd["folder_id"].asUUID()); const BOOL recursive = getIsRecursive(folder_id); fetcher->addRequestAtFront(folder_id, recursive, true); } -- cgit v1.3 From bbf9de9c6717f38a77a39d42d8493d275d558db9 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 26 Aug 2014 18:33:14 -0400 Subject: Bring better error handling to inventory item and folder fetching. First, introduced some LLSD-based interfaces to the llcorehttp code using utils classes (in llcorehttputil). I've kept LLSD out of the llcorehttp library up to now and will continue to do that. Functions provide a requestPost based on LLSD body and conversion utils for HttpResponse-to-LLSD and HttpResponse-to-string conversions. Inventory fetch operations now do more thorough error checking including 200-with-error status checking. Still do retry forever on folders though I don't like that. --- indra/llmessage/CMakeLists.txt | 4 + indra/llmessage/llcorehttputil.cpp | 137 ++++++++++++++++++++++ indra/llmessage/llcorehttputil.h | 115 ++++++++++++++++++ indra/newview/llinventorymodel.cpp | 76 +++++++----- indra/newview/llinventorymodelbackgroundfetch.cpp | 115 +++++++++++++----- 5 files changed, 386 insertions(+), 61 deletions(-) create mode 100644 indra/llmessage/llcorehttputil.cpp create mode 100644 indra/llmessage/llcorehttputil.h (limited to 'indra/newview/llinventorymodelbackgroundfetch.cpp') diff --git a/indra/llmessage/CMakeLists.txt b/indra/llmessage/CMakeLists.txt index 8bd134dc84..40eddcb0ab 100755 --- a/indra/llmessage/CMakeLists.txt +++ b/indra/llmessage/CMakeLists.txt @@ -6,6 +6,7 @@ include(00-Common) include(GoogleMock) include(LLAddBuildTest) include(LLCommon) +include(LLCoreHttp) include(LLMath) include(LLMessage) include(LLVFS) @@ -18,6 +19,7 @@ include_directories (${CMAKE_CURRENT_SOURCE_DIR}) include_directories( ${LLCOMMON_INCLUDE_DIRS} + ${LLCOREHTTP_INCLUDE_DIRS} ${LLMATH_INCLUDE_DIRS} ${LLMESSAGE_INCLUDE_DIRS} ${LLVFS_INCLUDE_DIRS} @@ -36,6 +38,7 @@ set(llmessage_SOURCE_FILES llchainio.cpp llcircuit.cpp llclassifiedflags.cpp + llcorehttputil.cpp llcurl.cpp lldatapacker.cpp lldispatcher.cpp @@ -124,6 +127,7 @@ set(llmessage_HEADER_FILES llcipher.h llcircuit.h llclassifiedflags.h + llcorehttputil.h llcurl.h lldatapacker.h lldbstrings.h diff --git a/indra/llmessage/llcorehttputil.cpp b/indra/llmessage/llcorehttputil.cpp new file mode 100644 index 0000000000..8d09aac971 --- /dev/null +++ b/indra/llmessage/llcorehttputil.cpp @@ -0,0 +1,137 @@ +/** + * @file llcorehttputil.cpp + * @date 2014-08-25 + * @brief Implementation of adapter and utility classes expanding the llcorehttp interfaces. + * + * $LicenseInfo:firstyear=2014&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "linden_common.h" + +#include + +#include "llcorehttputil.h" +#include "llsdserialize.h" + + +using namespace LLCore; + + +namespace LLCoreHttpUtil +{ + + +bool responseToLLSD(HttpResponse * response, bool log, LLSD & out_llsd) +{ + // Convert response to LLSD + BufferArray * body(response->getBody()); + if (! body || ! body->size()) + { + return false; + } + + LLCore::BufferArrayStream bas(body); + LLSD body_llsd; + S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas, log)); + if (LLSDParser::PARSE_FAILURE == parse_status){ + return false; + } + out_llsd = body_llsd; + return true; +} + + +HttpHandle requestPostWithLLSD(HttpRequest * request, + HttpRequest::policy_t policy_id, + HttpRequest::priority_t priority, + const std::string & url, + const LLSD & body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler) +{ + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + BufferArray * ba = new BufferArray(); + BufferArrayStream bas(ba); + LLSDSerialize::toXML(body, bas); + + handle = request->requestPost(policy_id, + priority, + url, + ba, + options, + headers, + handler); + ba->release(); + return handle; +} + + +std::string responseToString(LLCore::HttpResponse * response) +{ + static const std::string empty("[Empty]"); + + if (! response) + { + return empty; + } + + BufferArray * body(response->getBody()); + if (! body || ! body->size()) + { + return empty; + } + + // Attempt to parse as LLSD regardless of content-type + LLSD body_llsd; + if (responseToLLSD(response, false, body_llsd)) + { + std::ostringstream tmp; + + LLSDSerialize::toPrettyNotation(body_llsd, tmp); + std::size_t temp_len(tmp.tellp()); + + if (temp_len) + { + return tmp.str().substr(0, std::min(temp_len, std::size_t(1024))); + } + } + else + { + // *TODO: More elaborate forms based on Content-Type as needed. + char content[1024]; + + size_t len(body->read(0, content, sizeof(content))); + if (len) + { + return std::string(content, 0, len); + } + } + + // Default + return empty; +} + + +} // end namespace LLCoreHttpUtil + diff --git a/indra/llmessage/llcorehttputil.h b/indra/llmessage/llcorehttputil.h new file mode 100644 index 0000000000..d40172bc7a --- /dev/null +++ b/indra/llmessage/llcorehttputil.h @@ -0,0 +1,115 @@ +/** + * @file llcorehttputil.h + * @date 2014-08-25 + * @brief Adapter and utility classes expanding the llcorehttp interfaces. + * + * $LicenseInfo:firstyear=2014&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2014, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef LL_LLCOREHTTPUTIL_H +#define LL_LLCOREHTTPUTIL_H + +#include + +#include "httpcommon.h" +#include "httprequest.h" +#include "httpresponse.h" +#include "httpheaders.h" +#include "httpoptions.h" +#include "httphandler.h" +#include "bufferarray.h" +#include "bufferstream.h" +#include "llsd.h" + +/// +/// The base llcorehttp library implements many HTTP idioms +/// used in the viewer but not all. That library intentionally +/// avoids the use of LLSD and its conventions which aren't +/// universally applicable. This module, using namespace +/// LLCoreHttpUtil, provides the additional helper functions +/// that support idiomatic LLSD transport via the newer +/// llcorehttp library. +/// +namespace LLCoreHttpUtil +{ + +/// Attempt to convert a response object's contents to LLSD. +/// It is expected that the response body will be of non-zero +/// length on input but basic checks will be performed and +/// and error (false status) returned if there is no data. +/// If there is data but it cannot be successfully parsed, +/// an error is also returned. If successfully parsed, +/// the output LLSD object, out_llsd, is written with the +/// result and true is returned. +/// +/// @arg response Response object as returned in +/// in an HttpHandler onCompleted() callback. +/// @arg log If true, LLSD parser will emit errors +/// as LL_INFOS-level messages as it parses. +/// Otherwise, it *should* be a quiet parse. +/// @arg out_llsd Output LLSD object written only upon +/// successful parse of the response object. +/// +/// @return Returns true (and writes to out_llsd) if +/// parse was successful. False otherwise. +/// +bool responseToLLSD(LLCore::HttpResponse * response, + bool log, + LLSD & out_llsd); + +/// Create a std::string representation of a response object +/// suitable for logging. Mainly intended for logging of +/// failures and debug information. This won't be fast, +/// just adequate. +std::string responseToString(LLCore::HttpResponse * response); + + +/// Issue a standard HttpRequest::requestPost() call but using +/// and LLSD object as the request body. Conventions are the +/// same as with that method. Caller is expected to provide +/// an HttpHeaders object with a correct 'Content-Type:' header. +/// One will not be provided by this call. You might look after +/// the 'Accept:' header as well. +/// +/// @return If request is successfully issued, the +/// HttpHandle representing the request. +/// On error, LLCORE_HTTP_HANDLE_INVALID +/// is returned and caller can fetch detailed +/// status with the getStatus() method on the +/// request object. In case of error, no +/// request is queued and caller may need to +/// perform additional cleanup such as freeing +/// a now-useless HttpHandler object. +/// +LLCore::HttpHandle requestPostWithLLSD(LLCore::HttpRequest * request, + LLCore::HttpRequest::policy_t policy_id, + LLCore::HttpRequest::priority_t priority, + const std::string & url, + const LLSD & body, + LLCore::HttpOptions * options, + LLCore::HttpHeaders * headers, + LLCore::HttpHandler * handler); + +} // end namespace LLCoreHttpUtil + + +#endif // LL_LLCOREHTTPUTIL_H diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 2dd31f047f..5273fb6d96 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -55,7 +55,7 @@ #include "llsdutil.h" #include "bufferarray.h" #include "bufferstream.h" -#include "llsdserialize.h" +#include "llcorehttputil.h" //#define DIFF_INVENTORY_FILES #ifdef DIFF_INVENTORY_FILES @@ -75,12 +75,6 @@ BOOL LLInventoryModel::sFirstTimeInViewer2 = TRUE; static const char CACHE_FORMAT_STRING[] = "%s.inv"; static const char * const LOG_INV("Inventory"); -static std::string dumpResponse() -{ - return std::string("ADD SOMETHING MEANINGFUL HERE"); -} - - struct InventoryIDPtrLess { bool operator()(const LLViewerInventoryCategory* i1, const LLViewerInventoryCategory* i2) const @@ -2468,18 +2462,15 @@ LLCore::HttpHandle LLInventoryModel::requestPost(bool foreground, LLCore::HttpRequest * request(foreground ? mHttpRequestFG : mHttpRequestBG); LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(body, bas); - handle = request->requestPost(mHttpPolicyClass, - (foreground ? mHttpPriorityFG : mHttpPriorityBG), - url, - ba, - mHttpOptions, - mHttpHeaders, - handler); - ba->release(); + handle = LLCoreHttpUtil::requestPostWithLLSD(request, + mHttpPolicyClass, + (foreground ? mHttpPriorityFG : mHttpPriorityBG), + url, + body, + mHttpOptions, + mHttpHeaders, + handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { LLCore::HttpStatus status(request->getStatus()); @@ -3981,6 +3972,7 @@ void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle hand LLCore::HttpResponse * response) { LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); @@ -3988,23 +3980,41 @@ void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle hand else { LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling if (! body || ! body->size()) { LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; processFailure("HTTP response for inventory item query missing body", response); goto only_exit; } - - LLCore::BufferArrayStream bas(body); + + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling LLSD body_llsd; - S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas)); - if (LLSDParser::PARSE_FAILURE == parse_status) + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) { // INFOS-level logging will occur on the parsed failure processFailure("HTTP response for inventory item query has malformed LLSD", response); goto only_exit; } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + goto only_exit; + } + + // Check for 200-with-error failures + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + goto only_exit; + } + // Okay, process data if possible processData(body_llsd, response); } @@ -4016,12 +4026,6 @@ only_exit: void LLInventoryModel::FetchItemHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) { - if (! content.isMap()) - { - processFailure("LLSD response for inventory item not a map", response); - return; - } - start_new_inventory_observer(); #if 0 @@ -4083,7 +4087,7 @@ void LLInventoryModel::FetchItemHttpHandler::processData(LLSD & content, LLCore: { changes |= gInventory.updateItem(*it); } - // *HUH: Have computed changes, nothing uses it. + // *HUH: Have computed 'changes', nothing uses it. gInventory.notifyObservers(); gViewerWindow->getWindow()->decBusyCount(); @@ -4092,13 +4096,23 @@ void LLInventoryModel::FetchItemHttpHandler::processData(LLSD & content, LLCore: void LLInventoryModel::FetchItemHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) { - LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + const std::string & ct(response->getContentType()); + LL_WARNS(LOG_INV) << "Inventory item fetch failure\n" + << "[Status: " << status.toTerseString() << "]\n" + << "[Reason: " << status.toString() << "]\n" + << "[Content-type: " << ct << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; gInventory.notifyObservers(); } void LLInventoryModel::FetchItemHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) { - LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory item fetch failure\n" + << "[Status: internal error]\n" + << "[Reason: " << reason << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; gInventory.notifyObservers(); } diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 7b944edf45..0c04a9c039 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -40,17 +40,24 @@ #include "llhttpconstants.h" #include "bufferarray.h" #include "bufferstream.h" -#include "llsdserialize.h" +#include "llcorehttputil.h" namespace { +// // Http request handler class for single inventory item requests. // // We'll use a handler-per-request pattern here rather than // a shared handler. Mainly convenient as this was converted // from a Responder class model. // +// Derives from and is identical to the normal FetchItemHttpHandler +// except that: 1) it uses the background request object which is +// updated more slowly than the foreground and 2) keeps a count of +// active requests on the LLInventoryModelBackgroundFetch object +// to indicate outstanding operations are in-flight. +// class BGItemHttpHandler : public LLInventoryModel::FetchItemHttpHandler { LOG_CLASS(BGItemHttpHandler); @@ -74,6 +81,10 @@ protected: // Http request handler class for folders. +// +// Handler for FetchInventoryDescendents2 and FetchLibDescendents2 +// caps requests for folders. +// class BGFolderHttpHandler : public LLCore::HttpHandler { LOG_CLASS(BGFolderHttpHandler); @@ -116,12 +127,6 @@ const S32 MAX_FETCH_RETRIES = 10; const char * const LOG_INV("Inventory"); -std::string dumpResponse() -{ - return std::string("ADD SOMETHING MEANINGFUL HERE"); -} - - } // end of namespace anonymous @@ -471,6 +476,10 @@ void LLInventoryModelBackgroundFetch::bulkFetch() return; } + // *TODO: These values could be tweaked at runtime to effect + // a fast/slow fetch throttle. Once login is complete and the scene + // is mostly loaded, we could turn up the throttle and fill missing + // inventory more quickly. static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections static const F32 new_min_time(0.5f); // *HACK: Clean this up when old code goes away entirely. static const U32 max_batch_size(10); @@ -482,7 +491,7 @@ void LLInventoryModelBackgroundFetch::bulkFetch() if (mFetchCount) { - // Process completed HTTP requests + // Process completed background HTTP requests gInventory.handleResponses(false); } @@ -497,6 +506,8 @@ void LLInventoryModelBackgroundFetch::bulkFetch() const U32 sort_order(gSavedSettings.getU32(LLInventoryPanel::DEFAULT_SORT_ORDER) & 0x1); + // *TODO: Think I'd like to get a shared pointer to this and share it + // among all the folder requests. uuid_vec_t recursive_cats; LLSD folder_request_body; @@ -673,7 +684,9 @@ bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LL } +// =============================== // Anonymous Namespace Definitions +// =============================== namespace { @@ -683,30 +696,53 @@ namespace void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); } else { + // Response body should be present. LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling if (! body || ! body->size()) { LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; processFailure("HTTP response missing expected body", response); goto only_exit; } - - LLCore::BufferArrayStream bas(body); + + // Could test 'Content-Type' header but probably unreliable. + + // Convert response to LLSD + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling LLSD body_llsd; - S32 parse_status(LLSDSerialize::fromXML(body_llsd, bas)); - if (LLSDParser::PARSE_FAILURE == parse_status) + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) { // INFOS-level logging will occur on the parsed failure processFailure("HTTP response contained malformed LLSD", response); goto only_exit; } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response not a map", response); + goto only_exit; + } + + // Check for 200-with-error failures + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + goto only_exit; + } + // Okay, process data if possible processData(body_llsd, response); } @@ -719,13 +755,12 @@ only_exit: void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * response) { - if (! content.isMap()) - { - processFailure("LLSD response not a map", response); - return; - } - LLInventoryModelBackgroundFetch * fetcher(LLInventoryModelBackgroundFetch::getInstance()); + + // API V2 and earlier should probably be testing for "error" map + // in response as an application-level error. + + // Instead, we assume success and attempt to extract information. if (content.has("folders")) { LLSD folders(content["folders"]); @@ -836,7 +871,7 @@ void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * res folder_it != bad_folders.endArray(); ++folder_it) { - // *TODO: Stop copying data + // *TODO: Stop copying data [ed: this isn't copying data] LLSD folder_sd(*folder_it); // These folders failed on the dataserver. We probably don't want to retry them. @@ -857,14 +892,26 @@ void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * res void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::HttpResponse * response) { - LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + const std::string & ct(response->getContentType()); + LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n" + << "[Status: " << status.toTerseString() << "]\n" + << "[Reason: " << status.toString() << "]\n" + << "[Content-type: " << ct << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; + + // Could use a 404 test here to try to detect revoked caps... + + // This was originally the request retry logic for the inventory + // request which tested on HTTP_INTERNAL_ERROR status. This + // retry logic was unbounded and lacked discrimination as to the + // cause of the retry. The new http library should be doing + // adquately on retries but I want to keep the structure of a + // retry for reference. LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - - LL_INFOS(LOG_INV) << dumpResponse() << LL_ENDL; - - // *FIX: Not the correct test here... - if (status == LLCore::HttpStatus(408)) // timed out or curl failure + if (false) { + // timed out or curl failure for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); folder_it != mRequestSD["folders"].endArray(); ++folder_it) @@ -888,12 +935,20 @@ void BGFolderHttpHandler::processFailure(LLCore::HttpStatus status, LLCore::Http void BGFolderHttpHandler::processFailure(const char * const reason, LLCore::HttpResponse * response) { - LL_WARNS(LOG_INV) << dumpResponse() << LL_ENDL; + LL_WARNS(LOG_INV) << "Inventory folder fetch failure\n" + << "[Status: internal error]\n" + << "[Reason: " << reason << "]\n" + << "[Content (abridged): " + << LLCoreHttpUtil::responseToString(response) << "]" << LL_ENDL; + + // Reverse of previous processFailure() method, this is invoked + // when response structure is found to be invalid. Original + // always re-issued the request (without limit). This does + // the same but be aware that this may be a source of problems. + // Philosophy is that inventory folders are so essential to + // operation that this is a reasonable action. LLInventoryModelBackgroundFetch *fetcher = LLInventoryModelBackgroundFetch::getInstance(); - - LL_INFOS(LOG_INV) << dumpResponse() << LL_ENDL; - - if (true /* status == LLCore::HttpStatus(408)*/) // timed out or curl failure + if (true) { for (LLSD::array_const_iterator folder_it = mRequestSD["folders"].beginArray(); folder_it != mRequestSD["folders"].endArray(); -- cgit v1.3 From f71c6c745bc390fadc571801a0d7c043249ade24 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 9 Sep 2014 15:36:35 -0400 Subject: Cleanup pass. Documentation. Get older llcorehttp-using code to use utils for any LLSD interfaces. --- indra/llmessage/llcorehttputil.cpp | 4 +- indra/newview/llinventorymodel.cpp | 14 ++++- indra/newview/llinventorymodelbackgroundfetch.cpp | 28 +++++++--- indra/newview/llmeshrepository.cpp | 64 +++++++++-------------- indra/newview/lltexturefetch.cpp | 34 ++++++------ 5 files changed, 78 insertions(+), 66 deletions(-) (limited to 'indra/newview/llinventorymodelbackgroundfetch.cpp') diff --git a/indra/llmessage/llcorehttputil.cpp b/indra/llmessage/llcorehttputil.cpp index 8d09aac971..ee80b0fd94 100644 --- a/indra/llmessage/llcorehttputil.cpp +++ b/indra/llmessage/llcorehttputil.cpp @@ -39,7 +39,9 @@ using namespace LLCore; namespace LLCoreHttpUtil { - +// *TODO: Currently converts only from XML content. A mode +// to convert using fromBinary() might be useful as well. Mesh +// headers could use it. bool responseToLLSD(HttpResponse * response, bool log, LLSD & out_llsd) { // Convert response to LLSD diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 5273fb6d96..1625ae0d1b 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -3958,7 +3958,10 @@ BOOL decompress_file(const char* src_filename, const char* dst_filename) } #endif -// ==== FetchItemHttpHandler ==== + +///---------------------------------------------------------------------------- +/// Class LLInventoryModel::FetchItemHttpHandler +///---------------------------------------------------------------------------- LLInventoryModel::FetchItemHttpHandler::FetchItemHttpHandler(const LLSD & request_sd) : LLCore::HttpHandler(), @@ -4006,6 +4009,15 @@ void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle hand } // Check for 200-with-error failures + // + // Original Responder-based serivce model didn't check for these errors. + // It may be more robust to ignore this condition. With aggregated requests, + // an error in one inventory item might take down the entire request. + // So if this instead broke up the aggregated items into single requests, + // maybe that would make progress. Or perhaps there's structured information + // that can tell us what went wrong. Need to dig into this and firm up + // the API. + // // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling // body_llsd["error"]["identifier"] = "Development"; // body_llsd["error"]["message"] = "You left development code in the viewer"; diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 0c04a9c039..013134843a 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -45,6 +45,10 @@ namespace { +///---------------------------------------------------------------------------- +/// Class ::BGItemHttpHandler +///---------------------------------------------------------------------------- + // // Http request handler class for single inventory item requests. // @@ -80,6 +84,10 @@ protected: }; +///---------------------------------------------------------------------------- +/// Class ::BGFolderHttpHandler +///---------------------------------------------------------------------------- + // Http request handler class for folders. // // Handler for FetchInventoryDescendents2 and FetchLibDescendents2 @@ -130,6 +138,10 @@ const char * const LOG_INV("Inventory"); } // end of namespace anonymous +///---------------------------------------------------------------------------- +/// Class LLInventoryModelBackgroundFetch +///---------------------------------------------------------------------------- + LLInventoryModelBackgroundFetch::LLInventoryModelBackgroundFetch(): mBackgroundFetchActive(FALSE), mFolderFetchActive(false), @@ -684,14 +696,12 @@ bool LLInventoryModelBackgroundFetch::fetchQueueContainsNoDescendentsOf(const LL } -// =============================== -// Anonymous Namespace Definitions -// =============================== - namespace { -// ==== BGFolderHttpHandler ==== +///---------------------------------------------------------------------------- +/// Class ::BGFolderHttpHandler +///---------------------------------------------------------------------------- void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { @@ -734,6 +744,9 @@ void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRes } // Check for 200-with-error failures + // + // See comments in llinventorymodel.cpp about this mode of error. + // // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling // body_llsd["error"]["identifier"] = "Development"; // body_llsd["error"]["message"] = "You left development code in the viewer"; @@ -976,8 +989,9 @@ bool BGFolderHttpHandler::getIsRecursive(const LLUUID & cat_id) const return std::find(mRecursiveCatUUIDs.begin(), mRecursiveCatUUIDs.end(), cat_id) != mRecursiveCatUUIDs.end(); } - -// ==== BGItemHttpHandler ==== +///---------------------------------------------------------------------------- +/// Class ::BGItemHttpHandler +///---------------------------------------------------------------------------- // Nothing to implement here. All ctor/dtor changes. diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index a6707392fe..2b044c6916 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -71,6 +71,7 @@ #include "bufferarray.h" #include "bufferstream.h" #include "llfasttimer.h" +#include "llcorehttputil.h" #include "boost/lexical_cast.hpp" @@ -2236,21 +2237,17 @@ void LLMeshUploadThread::doWholeModelUpload() mModelData = LLSD::emptyMap(); wholeModelToLLSD(mModelData, true); LLSD body = mModelData["asset_resources"]; - dump_llsd_to_file(body,make_dump_name("whole_model_body_",dump_num)); - - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(body, bas); - // LLSDSerialize::toXML(mModelData, bas); // <- Enabling this will generate a convenient upload error - LLCore::HttpHandle handle = mHttpRequest->requestPost(mHttpPolicyClass, - mHttpPriority, - mWholeModelUploadURL, - ba, - mHttpOptions, - mHttpHeaders, - this); - ba->release(); - + + dump_llsd_to_file(body, make_dump_name("whole_model_body_", dump_num)); + + LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(mHttpRequest, + mHttpPolicyClass, + mHttpPriority, + mWholeModelUploadURL, + body, + mHttpOptions, + mHttpHeaders, + this); if (LLCORE_HTTP_HANDLE_INVALID == handle) { mHttpStatus = mHttpRequest->getStatus(); @@ -2294,19 +2291,14 @@ void LLMeshUploadThread::requestWholeModelFee() mModelData = LLSD::emptyMap(); wholeModelToLLSD(mModelData, false); dump_llsd_to_file(mModelData, make_dump_name("whole_model_fee_request_", dump_num)); - - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(mModelData, bas); - - LLCore::HttpHandle handle = mHttpRequest->requestPost(mHttpPolicyClass, - mHttpPriority, - mWholeModelFeeCapability, - ba, - mHttpOptions, - mHttpHeaders, - this); - ba->release(); + LLCore::HttpHandle handle = LLCoreHttpUtil::requestPostWithLLSD(mHttpRequest, + mHttpPolicyClass, + mHttpPriority, + mWholeModelFeeCapability, + mModelData, + mHttpOptions, + mHttpHeaders, + this); if (LLCORE_HTTP_HANDLE_INVALID == handle) { mHttpStatus = mHttpRequest->getStatus(); @@ -2379,12 +2371,8 @@ void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResp } else { - LLCore::BufferArray * ba(response->getBody()); - if (ba && ba->size()) - { - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::fromXML(body, bas); - } + // *TODO: handle error in conversion process + LLCoreHttpUtil::responseToLLSD(response, true, body); } dump_llsd_to_file(body, make_dump_name("whole_model_upload_response_", dump_num)); @@ -2443,12 +2431,8 @@ void LLMeshUploadThread::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResp } else { - LLCore::BufferArray * ba(response->getBody()); - if (ba && ba->size()) - { - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::fromXML(body, bas); - } + // *TODO: handle error in conversion process + LLCoreHttpUtil::responseToLLSD(response, true, body); } dump_llsd_to_file(body, make_dump_name("whole_model_fee_response_", dump_num)); diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index a64a6ee091..a955b907d4 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -56,13 +56,13 @@ #include "llsdparam.h" #include "llsdutil.h" #include "llstartup.h" -#include "llsdserialize.h" #include "httprequest.h" #include "httphandler.h" #include "httpresponse.h" #include "bufferarray.h" #include "bufferstream.h" +#include "llcorehttputil.h" #include "llhttpretrypolicy.h" @@ -1349,20 +1349,20 @@ bool LLTextureFetchWorker::doWork(S32 param) LL_WARNS(LOG_TXT) << "trying to seek a non-default texture on the sim. Bad!" << LL_ENDL; } setUrl(http_url + "/?texture_id=" + mID.asString().c_str()); - LL_DEBUGS("Texture") << "Texture URL " << mUrl << LL_ENDL; + LL_DEBUGS(LOG_TXT) << "Texture URL: " << mUrl << LL_ENDL; mWriteToCacheState = CAN_WRITE ; //because this texture has a fixed texture id. } else { mCanUseHTTP = false ; - LL_DEBUGS("Texture") << "Texture not available via HTTP: no URL " << mUrl << LL_ENDL; + LL_DEBUGS(LOG_TXT) << "Texture not available via HTTP: empty URL." << LL_ENDL; } } else { // This will happen if not logged in or if a region deoes not have HTTP Texture enabled //LL_WARNS(LOG_TXT) << "Region not found for host: " << mHost << LL_ENDL; - LL_DEBUGS("Texture") << "Texture not available via HTTP: no region " << mUrl << LL_ENDL; + LL_DEBUGS(LOG_TXT) << "Texture not available via HTTP: no region " << mUrl << LL_ENDL; mCanUseHTTP = false; } } @@ -4026,7 +4026,9 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) // Update sequence number if (S32_MAX == ++report_sequence) + { report_sequence = 0; + } reporting_started = true; // Limit the size of the stats report if necessary. @@ -4035,18 +4037,16 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) if (! mCapsURL.empty()) { - LLCore::BufferArray * ba = new LLCore::BufferArray; - LLCore::BufferArrayStream bas(ba); - LLSDSerialize::toXML(sd, bas); - - fetcher->getHttpRequest().requestPost(fetcher->getPolicyClass(), - report_priority, - mCapsURL, - ba, - NULL, - fetcher->getMetricsHeaders(), - handler); - ba->release(); + // *TODO: Move this to a different class that expects POSTs sometime. + // Don't care about handle, this is a fire-and-forget operation. + LLCoreHttpUtil::requestPostWithLLSD(&fetcher->getHttpRequest(), + fetcher->getPolicyClass(), + report_priority, + mCapsURL, + sd, + NULL, + fetcher->getMetricsHeaders(), + handler); LLTextureFetch::svMetricsDataBreak = false; } else @@ -4057,7 +4057,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) // In QA mode, Metrics submode, log the result for ease of testing if (fetcher->isQAMode()) { - LL_INFOS("Textures") << ll_pretty_print_sd(sd) << LL_ENDL; + LL_INFOS(LOG_TXT) << ll_pretty_print_sd(sd) << LL_ENDL; } return true; -- cgit v1.3 From 11036d7bf471953ada9b877b8d9ce9de4b94dc5b Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 19 Sep 2014 19:43:25 -0400 Subject: Cleanup work. Use http constants for content-type and accept headers in mesh and textures. For texture metrics reporting, use the AP_INVENTORY policy class which is non-pipelined and pointing (usually) in the right direction. Use a do-while(false) structure to manage common exit path code in onCompleted() methods. Identical to a 'goto' but might amuse the pedantic. Tuning on background fetch to have it cycle faster. This is experimental. I suspect with HTTP balancing in llcorehttp, we can do away with the timers here. --- indra/llmessage/llhttpconstants.cpp | 6 +- indra/llmessage/llhttpconstants.h | 3 +- indra/newview/llappcorehttp.h | 9 +- indra/newview/llinventorymodel.cpp | 102 +++++++++++---------- indra/newview/llinventorymodelbackgroundfetch.cpp | 106 +++++++++++----------- indra/newview/llmeshrepository.cpp | 4 +- indra/newview/lltexturefetch.cpp | 15 +-- indra/newview/lltexturefetch.h | 6 +- 8 files changed, 130 insertions(+), 121 deletions(-) (limited to 'indra/newview/llinventorymodelbackgroundfetch.cpp') diff --git a/indra/llmessage/llhttpconstants.cpp b/indra/llmessage/llhttpconstants.cpp index 01f4a080b0..32f76f0d70 100755 --- a/indra/llmessage/llhttpconstants.cpp +++ b/indra/llmessage/llhttpconstants.cpp @@ -3,11 +3,8 @@ * @brief Implementation of the HTTP request / response constant lookups * * $LicenseInfo:firstyear=2013&license=viewerlgpl$ - * - * Copyright (c) 2013, Linden Research, Inc. - * * Second Life Viewer Source Code - * Copyright (C) 2013, Linden Research, Inc. + * Copyright (C) 2013-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -107,6 +104,7 @@ const std::string HTTP_IN_HEADER_X_FORWARDED_FOR("x-forwarded-for"); const std::string HTTP_CONTENT_LLSD_XML("application/llsd+xml"); const std::string HTTP_CONTENT_OCTET_STREAM("application/octet-stream"); +const std::string HTTP_CONTENT_VND_LL_MESH("application/vnd.ll.mesh"); const std::string HTTP_CONTENT_XML("application/xml"); const std::string HTTP_CONTENT_JSON("application/json"); const std::string HTTP_CONTENT_TEXT_HTML("text/html"); diff --git a/indra/llmessage/llhttpconstants.h b/indra/llmessage/llhttpconstants.h index 4aa3cc6394..d6bcbd3c19 100755 --- a/indra/llmessage/llhttpconstants.h +++ b/indra/llmessage/llhttpconstants.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2001&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2001-2013, Linden Research, Inc. + * Copyright (C) 2001-2014, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -203,6 +203,7 @@ extern const std::string HTTP_IN_HEADER_X_FORWARDED_FOR; extern const std::string HTTP_CONTENT_LLSD_XML; extern const std::string HTTP_CONTENT_OCTET_STREAM; +extern const std::string HTTP_CONTENT_VND_LL_MESH; extern const std::string HTTP_CONTENT_XML; extern const std::string HTTP_CONTENT_JSON; extern const std::string HTTP_CONTENT_TEXT_HTML; diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index e822a40300..37d7a737e7 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -66,7 +66,7 @@ public: /// baking service. Deep queueing of requests. /// Do not share. GET requests only. /// - /// Destination: simhost:12046 & bake-texture:80 + /// Destination: simhost:12046 & {bake-texture,cdn}:80 /// Protocol: http: /// Transfer size: KB-MB /// Long poll: no @@ -95,7 +95,7 @@ public: /// Very deeply queued. Do not share. GET /// requests only. /// - /// Destination: simhost:12046 + /// Destination: simhost:12046 & cdn:80 /// Protocol: http: /// Transfer size: KB-MB /// Long poll: no @@ -113,7 +113,7 @@ public: /// traffic that can wait for longish stalls /// (default timeout 600S). /// - /// Destination: simhost:12046 + /// Destination: simhost:12046 & cdn:80 /// Protocol: http: /// Transfer size: MB /// Long poll: no @@ -163,7 +163,8 @@ public: /// Request rate: high /// Pipelined: no AP_INVENTORY, - + AP_REPORTING = AP_INVENTORY, // Piggy-back on inventory + AP_COUNT // Must be last }; diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index 1625ae0d1b..ee28cef640 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -3974,64 +3974,68 @@ LLInventoryModel::FetchItemHttpHandler::~FetchItemHttpHandler() void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - LLCore::HttpStatus status(response->getStatus()); - // status = LLCore::HttpStatus(404); // Dev tool to force error handling - if (! status) - { - processFailure(status, response); - } - else + // Single-pass do-while used for common exit handling + do { - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) + LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling + if (! status) { - LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; - processFailure("HTTP response for inventory item query missing body", response); - goto only_exit; + processFailure(status, response); } - - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + else { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response for inventory item query has malformed LLSD", response); - goto only_exit; - } + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; + processFailure("HTTP response for inventory item query missing body", response); + break; // Goto common exit + } - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response for inventory item not a map", response); - goto only_exit; - } + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response for inventory item query has malformed LLSD", response); + break; // Goto common exit + } - // Check for 200-with-error failures - // - // Original Responder-based serivce model didn't check for these errors. - // It may be more robust to ignore this condition. With aggregated requests, - // an error in one inventory item might take down the entire request. - // So if this instead broke up the aggregated items into single requests, - // maybe that would make progress. Or perhaps there's structured information - // that can tell us what went wrong. Need to dig into this and firm up - // the API. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - goto only_exit; - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + break; // Goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Check for 200-with-error failures + // + // Original Responder-based serivce model didn't check for these errors. + // It may be more robust to ignore this condition. With aggregated requests, + // an error in one inventory item might take down the entire request. + // So if this instead broke up the aggregated items into single requests, + // maybe that would make progress. Or perhaps there's structured information + // that can tell us what went wrong. Need to dig into this and firm up + // the API. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // Goto common exit + } + + // Okay, process data if possible + processData(body_llsd, response); + } } + while (false); -only_exit: // Must delete on completion. delete this; } diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index 013134843a..de1d123fe5 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -288,6 +288,7 @@ void LLInventoryModelBackgroundFetch::setAllFoldersFetched() } mFolderFetchActive = false; mBackgroundFetchActive = false; + LL_INFOS(LOG_INV) << "Inventory background fetch completed" << LL_ENDL; } void LLInventoryModelBackgroundFetch::backgroundFetchCB(void *) @@ -314,12 +315,7 @@ void LLInventoryModelBackgroundFetch::backgroundFetch() // No more categories to fetch, stop fetch process. if (mFetchQueue.empty()) { - LL_INFOS(LOG_INV) << "Inventory fetch completed" << LL_ENDL; - setAllFoldersFetched(); - mBackgroundFetchActive = false; - mFolderFetchActive = false; - return; } @@ -493,9 +489,10 @@ void LLInventoryModelBackgroundFetch::bulkFetch() // is mostly loaded, we could turn up the throttle and fill missing // inventory more quickly. static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections - static const F32 new_min_time(0.5f); // *HACK: Clean this up when old code goes away entirely. + static const F32 new_min_time(0.05f); // *HACK: Clean this up when old code goes away entirely. static const U32 max_batch_size(10); + mMinTimeBetweenFetches = 0.01f; if (mMinTimeBetweenFetches < new_min_time) { mMinTimeBetweenFetches = new_min_time; // *HACK: See above. @@ -705,62 +702,66 @@ namespace void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - LLCore::HttpStatus status(response->getStatus()); - // status = LLCore::HttpStatus(404); // Dev tool to force error handling - if (! status) - { - processFailure(status, response); - } - else + // Single-pass do-while used for common exit handling + do { - // Response body should be present. - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) + LLCore::HttpStatus status(response->getStatus()); + // status = LLCore::HttpStatus(404); // Dev tool to force error handling + if (! status) { - LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; - processFailure("HTTP response missing expected body", response); - goto only_exit; + processFailure(status, response); } + else + { + // Response body should be present. + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; + processFailure("HTTP response missing expected body", response); + break; // Goto common exit + } - // Could test 'Content-Type' header but probably unreliable. + // Could test 'Content-Type' header but probably unreliable. - // Convert response to LLSD - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) - { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response contained malformed LLSD", response); - goto only_exit; - } + // Convert response to LLSD + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response contained malformed LLSD", response); + break; // goto common exit + } - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response not a map", response); - goto only_exit; - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response not a map", response); + break; // goto common exit + } - // Check for 200-with-error failures - // - // See comments in llinventorymodel.cpp about this mode of error. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - goto only_exit; - } + // Check for 200-with-error failures + // + // See comments in llinventorymodel.cpp about this mode of error. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Okay, process data if possible + processData(body_llsd, response); + } } + while (false); -only_exit: // Must delete on completion. delete this; } @@ -895,7 +896,6 @@ void BGFolderHttpHandler::processData(LLSD & content, LLCore::HttpResponse * res if (fetcher->isBulkFetchProcessingComplete()) { - LL_INFOS(LOG_INV) << "Inventory fetch completed" << LL_ENDL; fetcher->setAllFoldersFetched(); } diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 2b044c6916..648056484e 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -760,7 +760,7 @@ LLMeshRepoThread::LLMeshRepoThread() mHttpLargeOptions->setTransferTimeout(LARGE_MESH_XFER_TIMEOUT); mHttpLargeOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "application/vnd.ll.mesh"); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_VND_LL_MESH); mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH2); mHttpLegacyPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_MESH1); mHttpLargePolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_LARGE_MESH); @@ -1887,7 +1887,7 @@ LLMeshUploadThread::LLMeshUploadThread(LLMeshUploadThread::instance_list& data, mHttpOptions->setUseRetryAfter(gSavedSettings.getBOOL("MeshUseHttpRetryAfter")); mHttpOptions->setRetries(UPLOAD_RETRY_LIMIT); mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Content-Type", "application/llsd+xml"); + mHttpHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_UPLOADS); mHttpPriority = 0; } diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index a955b907d4..acd4cf2d8d 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -2514,8 +2514,9 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpOptions(NULL), mHttpOptionsWithHeaders(NULL), mHttpHeaders(NULL), - mHttpMetricsHeaders(NULL), mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpMetricsHeaders(NULL), + mHttpMetricsPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), mTotalCacheReadCount(0U), mTotalCacheWriteCount(0U), mTotalResourceWaitCount(0U), @@ -2528,15 +2529,16 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), U32Bytes(gSavedSettings.getU32("TextureLoggingThreshold"))); LLAppCoreHttp & app_core_http(LLAppViewer::instance()->getAppCoreHttp()); - mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); mHttpRequest = new LLCore::HttpRequest; mHttpOptions = new LLCore::HttpOptions; mHttpOptionsWithHeaders = new LLCore::HttpOptions; mHttpOptionsWithHeaders->setWantHeaders(true); mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "image/x-j2c"); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_IMAGE_X_J2C); + mHttpPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_TEXTURE); mHttpMetricsHeaders = new LLCore::HttpHeaders; - mHttpMetricsHeaders->append("Content-Type", "application/llsd+xml"); + mHttpMetricsHeaders->append(HTTP_OUT_HEADER_CONTENT_TYPE, HTTP_CONTENT_LLSD_XML); + mHttpMetricsPolicyClass = app_core_http.getPolicy(LLAppCoreHttp::AP_REPORTING); mHttpHighWater = HTTP_NONPIPE_REQUESTS_HIGH_WATER; mHttpLowWater = HTTP_NONPIPE_REQUESTS_LOW_WATER; mHttpSemaphore = 0; @@ -4037,10 +4039,9 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) if (! mCapsURL.empty()) { - // *TODO: Move this to a different class that expects POSTs sometime. // Don't care about handle, this is a fire-and-forget operation. LLCoreHttpUtil::requestPostWithLLSD(&fetcher->getHttpRequest(), - fetcher->getPolicyClass(), + fetcher->getMetricsPolicyClass(), report_priority, mCapsURL, sd, @@ -4227,7 +4228,7 @@ void LLTextureFetchDebugger::init() if (! mHttpHeaders) { mHttpHeaders = new LLCore::HttpHeaders; - mHttpHeaders->append("Accept", "image/x-j2c"); + mHttpHeaders->append(HTTP_OUT_HEADER_ACCEPT, HTTP_CONTENT_IMAGE_X_J2C); } } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 89d18e2c67..27779a31e0 100755 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -179,6 +179,9 @@ public: // Threads: T* LLCore::HttpHeaders * getMetricsHeaders() const { return mHttpMetricsHeaders; } + // Threads: T* + LLCore::HttpRequest::policy_t getMetricsPolicyClass() const { return mHttpMetricsPolicyClass; } + bool isQAMode() const { return mQAMode; } // ---------------------------------- @@ -354,8 +357,9 @@ private: LLCore::HttpOptions * mHttpOptions; // Ttf LLCore::HttpOptions * mHttpOptionsWithHeaders; // Ttf LLCore::HttpHeaders * mHttpHeaders; // Ttf - LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf LLCore::HttpRequest::policy_t mHttpPolicyClass; // T* + LLCore::HttpHeaders * mHttpMetricsHeaders; // Ttf + LLCore::HttpRequest::policy_t mHttpMetricsPolicyClass; // T* S32 mHttpHighWater; // Ttf S32 mHttpLowWater; // Ttf -- cgit v1.3 From 329608d24668b044e16b54ff7a7d0ac592b2b88d Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 22 Sep 2014 18:49:45 -0400 Subject: Tuning and documentation. Use a fast poll frequency (0.05S) on the HTTP requests for inventory. We'll benchmark with that and see how it goes. Document some of the history of the background fetcher for future devs. Suggest some future projects to make things faster. Pointers on using LLSD with the llcorehttp library in the readme. And restructured the LLSD onCompleted() processing phases using do{}while(false) which produced a code flow that is fairly attractive. --- indra/llcorehttp/README.Linden | 8 ++ indra/newview/llinventorymodel.cpp | 92 +++++++------- indra/newview/llinventorymodelbackgroundfetch.cpp | 140 +++++++++++++++------- 3 files changed, 147 insertions(+), 93 deletions(-) (limited to 'indra/newview/llinventorymodelbackgroundfetch.cpp') diff --git a/indra/llcorehttp/README.Linden b/indra/llcorehttp/README.Linden index eb6ccab3bc..c3aaa9158d 100644 --- a/indra/llcorehttp/README.Linden +++ b/indra/llcorehttp/README.Linden @@ -529,6 +529,14 @@ HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148 data = NULL; + There are now helper functions in llmessage/llcorehttputil.h to + assist with LLSD usage. requestPostWithLLSD(...) provides a + requestPost()-like interface that takes an LLSD object rather than + a BufferArray. And responseToLLSD(...) attempts to convert a + BufferArray received from a server into an LLSD object. You can + find examples in llmeshrepository.cpp, llinventorymodel.cpp, + llinventorymodelbackgroundfetch.cpp and lltexturefetch.cpp. + LLSD will often go hand-in-hand with BufferArray and data transport. But you can also do all the streaming I/O you'd expect of a std::iostream object: diff --git a/indra/newview/llinventorymodel.cpp b/indra/newview/llinventorymodel.cpp index ee28cef640..dab3a4c06d 100755 --- a/indra/newview/llinventorymodel.cpp +++ b/indra/newview/llinventorymodel.cpp @@ -3974,65 +3974,63 @@ LLInventoryModel::FetchItemHttpHandler::~FetchItemHttpHandler() void LLInventoryModel::FetchItemHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - // Single-pass do-while used for common exit handling - do + do // Single-pass do-while used for common exit handling { LLCore::HttpStatus status(response->getStatus()); // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); + break; // Goto common exit } - else - { - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) - { - LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; - processFailure("HTTP response for inventory item query missing body", response); - break; // Goto common exit - } - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) - { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response for inventory item query has malformed LLSD", response); - break; // Goto common exit - } + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory item query." << LL_ENDL; + processFailure("HTTP response for inventory item query missing body", response); + break; // Goto common exit + } - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response for inventory item not a map", response); - break; // Goto common exit - } + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response for inventory item query has malformed LLSD", response); + break; // Goto common exit + } - // Check for 200-with-error failures - // - // Original Responder-based serivce model didn't check for these errors. - // It may be more robust to ignore this condition. With aggregated requests, - // an error in one inventory item might take down the entire request. - // So if this instead broke up the aggregated items into single requests, - // maybe that would make progress. Or perhaps there's structured information - // that can tell us what went wrong. Need to dig into this and firm up - // the API. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - break; // Goto common exit - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response for inventory item not a map", response); + break; // Goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Check for 200-with-error failures + // + // Original Responder-based serivce model didn't check for these errors. + // It may be more robust to ignore this condition. With aggregated requests, + // an error in one inventory item might take down the entire request. + // So if this instead broke up the aggregated items into single requests, + // maybe that would make progress. Or perhaps there's structured information + // that can tell us what went wrong. Need to dig into this and firm up + // the API. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // Goto common exit } + + // Okay, process data if possible + processData(body_llsd, response); } while (false); diff --git a/indra/newview/llinventorymodelbackgroundfetch.cpp b/indra/newview/llinventorymodelbackgroundfetch.cpp index de1d123fe5..f18832fe95 100755 --- a/indra/newview/llinventorymodelbackgroundfetch.cpp +++ b/indra/newview/llinventorymodelbackgroundfetch.cpp @@ -42,6 +42,56 @@ #include "bufferstream.h" #include "llcorehttputil.h" +// History (may be apocryphal) +// +// Around V2, an HTTP inventory download mechanism was added +// along with inventory LINK items referencing other inventory +// items. As part of this, at login, the entire inventory +// structure is downloaded 'in the background' using the +// backgroundFetch()/bulkFetch() methods. The UDP path can +// still be used and is found in the 'DEPRECATED OLD CODE' +// section. +// +// The old UDP path implemented a throttle that adapted +// itself during running. The mechanism survived info HTTP +// somewhat but was pinned to poll the HTTP plumbing at +// 0.5S intervals. The reasons for this particular value +// have been lost. It's possible to switch between UDP +// and HTTP while this is happening but there may be +// surprises in what happens in that case. +// +// Conversion to llcorehttp reduced the number of connections +// used but batches more data and queues more requests (but +// doesn't due pipelining due to libcurl restrictions). The +// poll interval above was re-examined and reduced to get +// inventory into the viewer more quickly. +// +// Possible future work: +// +// * Don't download the entire heirarchy in one go (which +// might have been how V1 worked). Implications for +// links (which may not have a valid target) and search +// which would then be missing data. +// +// * Review the download rate throttling. Slow then fast? +// Detect bandwidth usage and speed up when it drops? +// +// * A lot of calls to notifyObservers(). It looks like +// these could be collapsed by maintaining a 'dirty' +// bit and there appears to be an attempt to do this. +// But it isn't used or is used in a limited fashion. +// Are there semanic issues requiring a call after certain +// updateItem() calls? +// +// * An error on a fetch could be due to one item in the batch. +// If the batch were broken up, perhaps more of the inventory +// would download. (Handwave here, not certain this is an +// issue in practice.) +// +// * Conversion to AISv3. +// + + namespace { @@ -488,11 +538,11 @@ void LLInventoryModelBackgroundFetch::bulkFetch() // a fast/slow fetch throttle. Once login is complete and the scene // is mostly loaded, we could turn up the throttle and fill missing // inventory more quickly. + static const U32 max_batch_size(10); static const S32 max_concurrent_fetches(12); // Outstanding requests, not connections static const F32 new_min_time(0.05f); // *HACK: Clean this up when old code goes away entirely. - static const U32 max_batch_size(10); - mMinTimeBetweenFetches = 0.01f; + mMinTimeBetweenFetches = new_min_time; if (mMinTimeBetweenFetches < new_min_time) { mMinTimeBetweenFetches = new_min_time; // *HACK: See above. @@ -702,63 +752,61 @@ namespace void BGFolderHttpHandler::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) { - // Single-pass do-while used for common exit handling - do + do // Single-pass do-while used for common exit handling { LLCore::HttpStatus status(response->getStatus()); // status = LLCore::HttpStatus(404); // Dev tool to force error handling if (! status) { processFailure(status, response); + break; // Goto common exit } - else - { - // Response body should be present. - LLCore::BufferArray * body(response->getBody()); - // body = NULL; // Dev tool to force error handling - if (! body || ! body->size()) - { - LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; - processFailure("HTTP response missing expected body", response); - break; // Goto common exit - } - // Could test 'Content-Type' header but probably unreliable. + // Response body should be present. + LLCore::BufferArray * body(response->getBody()); + // body = NULL; // Dev tool to force error handling + if (! body || ! body->size()) + { + LL_WARNS(LOG_INV) << "Missing data in inventory folder query." << LL_ENDL; + processFailure("HTTP response missing expected body", response); + break; // Goto common exit + } - // Convert response to LLSD - // body->write(0, "Garbage Response", 16); // Dev tool to force error handling - LLSD body_llsd; - if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) - { - // INFOS-level logging will occur on the parsed failure - processFailure("HTTP response contained malformed LLSD", response); - break; // goto common exit - } + // Could test 'Content-Type' header but probably unreliable. - // Expect top-level structure to be a map - // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling - if (! body_llsd.isMap()) - { - processFailure("LLSD response not a map", response); - break; // goto common exit - } + // Convert response to LLSD + // body->write(0, "Garbage Response", 16); // Dev tool to force error handling + LLSD body_llsd; + if (! LLCoreHttpUtil::responseToLLSD(response, true, body_llsd)) + { + // INFOS-level logging will occur on the parsed failure + processFailure("HTTP response contained malformed LLSD", response); + break; // goto common exit + } - // Check for 200-with-error failures - // - // See comments in llinventorymodel.cpp about this mode of error. - // - // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling - // body_llsd["error"]["identifier"] = "Development"; - // body_llsd["error"]["message"] = "You left development code in the viewer"; - if (body_llsd.has("error")) - { - processFailure("Inventory application error (200-with-error)", response); - break; // goto common exit - } + // Expect top-level structure to be a map + // body_llsd = LLSD::emptyArray(); // Dev tool to force error handling + if (! body_llsd.isMap()) + { + processFailure("LLSD response not a map", response); + break; // goto common exit + } - // Okay, process data if possible - processData(body_llsd, response); + // Check for 200-with-error failures + // + // See comments in llinventorymodel.cpp about this mode of error. + // + // body_llsd["error"] = LLSD::emptyMap(); // Dev tool to force error handling + // body_llsd["error"]["identifier"] = "Development"; + // body_llsd["error"]["message"] = "You left development code in the viewer"; + if (body_llsd.has("error")) + { + processFailure("Inventory application error (200-with-error)", response); + break; // goto common exit } + + // Okay, process data if possible + processData(body_llsd, response); } while (false); -- cgit v1.3