From 5611cb6d476540e6a1c654c1f9acdce2787b3505 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 23 Apr 2012 16:19:39 -0400 Subject: Okay, imported the core-http library and got it compiling suspiciously easily. The unit/integration tests don't work yet as I'm still battling cmake/autobuild as usual but first milestone passed. --- indra/llcorehttp/httpcommon.cpp | 76 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 indra/llcorehttp/httpcommon.cpp (limited to 'indra/llcorehttp/httpcommon.cpp') diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp new file mode 100644 index 0000000000..c37d081150 --- /dev/null +++ b/indra/llcorehttp/httpcommon.cpp @@ -0,0 +1,76 @@ +/** + * @file httpcommon.cpp + * @brief + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "httpcommon.h" + +#include +#include + + +namespace LLCore +{ + +HttpStatus::type_enum_t EXT_CURL_EASY; +HttpStatus::type_enum_t EXT_CURL_MULTI; +HttpStatus::type_enum_t LLCORE; + +std::string HttpStatus::toString() const +{ + static const char * llcore_errors[] = + { + "", + "Services shutting down", + "Operation canceled", + "Invalid Content-Range header encountered" + }; + static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); + + if (*this) + { + return std::string(""); + } + + switch (mType) + { + case EXT_CURL_EASY: + return std::string(curl_easy_strerror(CURLcode(mStatus))); + + case EXT_CURL_MULTI: + return std::string(curl_multi_strerror(CURLMcode(mStatus))); + + case LLCORE: + if (mStatus >= 0 && mStatus < llcore_errors_count) + { + return std::string(llcore_errors[mStatus]); + } + break; + + } + return std::string("Unknown error"); +} + +} // end namespace LLCore + -- cgit v1.3 From 8fc350125c671baeae6b7f8b1814251009f4f50a Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 23 May 2012 19:12:09 -0400 Subject: Integrate llcorehttp library into lltexturefetch design. This is the first functional viewer pass with the HTTP work of the texture fetch code performed by the llcorehttp library. Not exactly a 'drop-in' replacement but a work-alike with some changes (e.g. handler notification in consumer thread versus responder notification in worker thread). This also includes some temporary changes in the priority scheme to prevent the kind of priority inversion found in VWR-28996. Scheme used here does provide liveness if not optimal responsiveness or order-of-operation. The llcorehttp library at this point is far from optimally performing. Its worker thread is making relatively poor use of cycles it gets and it doesn't idle or sleep intelligently yet. This early integration step helps shake out the interfaces, implementation niceties will be covered soon. --- indra/llcorehttp/CMakeLists.txt | 2 + indra/llcorehttp/_httplibcurl.cpp | 11 +- indra/llcorehttp/_httplibcurl.h | 3 +- indra/llcorehttp/_httpopcancel.cpp | 82 ++++ indra/llcorehttp/_httpopcancel.h | 75 ++++ indra/llcorehttp/_httpoperation.h | 25 -- indra/llcorehttp/_httpoprequest.cpp | 105 +++-- indra/llcorehttp/_httpoprequest.h | 9 +- indra/llcorehttp/_httppolicy.h | 3 +- indra/llcorehttp/_httpreplyqueue.h | 2 +- indra/llcorehttp/_httprequestqueue.h | 2 +- indra/llcorehttp/_httpservice.h | 4 +- indra/llcorehttp/httpcommon.cpp | 90 ++++- indra/llcorehttp/httpcommon.h | 59 ++- indra/llcorehttp/httprequest.cpp | 17 + indra/llcorehttp/httpresponse.cpp | 3 +- indra/llcorehttp/httpresponse.h | 32 +- indra/llcorehttp/tests/test_httpstatus.hpp | 102 +++++ indra/newview/llappviewer.cpp | 159 ++++++++ indra/newview/lltexturefetch.cpp | 610 +++++++++++++++++------------ indra/newview/lltexturefetch.h | 19 +- 21 files changed, 1057 insertions(+), 357 deletions(-) create mode 100644 indra/llcorehttp/_httpopcancel.cpp create mode 100644 indra/llcorehttp/_httpopcancel.h (limited to 'indra/llcorehttp/httpcommon.cpp') diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt index 81c502b642..ae92fb96fd 100644 --- a/indra/llcorehttp/CMakeLists.txt +++ b/indra/llcorehttp/CMakeLists.txt @@ -30,6 +30,7 @@ set(llcorehttp_SOURCE_FILES _httprequestqueue.cpp _httpoperation.cpp _httpoprequest.cpp + _httpopcancel.cpp _httpreplyqueue.cpp _httppolicy.cpp _httplibcurl.cpp @@ -49,6 +50,7 @@ set(llcorehttp_HEADER_FILES httpresponse.h _httpoperation.h _httpoprequest.h + _httpopcancel.h _httprequestqueue.h _httpreplyqueue.h _httpservice.h diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 15be977adf..1b951818e4 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -27,7 +27,6 @@ #include "_httplibcurl.h" #include "httpheaders.h" - #include "_httpoprequest.h" #include "_httpservice.h" @@ -173,8 +172,11 @@ void HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode { int http_status(200); - curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &status); - op->mReplyStatus = http_status; + curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &http_status); + op->mStatus = LLCore::HttpStatus(http_status, + (http_status >= 200 && http_status <= 299 + ? HE_SUCCESS + : HE_REPLY_ERROR)); } // Detach from multi and recycle handle @@ -202,7 +204,8 @@ int HttpLibcurl::activeCount() const struct curl_slist * append_headers_to_slist(const HttpHeaders * headers, struct curl_slist * slist) { for (HttpHeaders::container_t::const_iterator it(headers->mHeaders.begin()); - headers->mHeaders.end() != it; + + headers->mHeaders.end() != it; ++it) { slist = curl_slist_append(slist, (*it).c_str()); diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 01c68320af..807196628d 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -45,6 +45,7 @@ class HttpOpRequest; class HttpHeaders; +/// Implements libcurl-based transport for an HttpService instance. class HttpLibcurl { public: @@ -71,7 +72,7 @@ protected: typedef std::set active_set_t; protected: - HttpService * mService; + HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; CURLM * mMultiHandles[1]; }; // end class HttpLibcurl diff --git a/indra/llcorehttp/_httpopcancel.cpp b/indra/llcorehttp/_httpopcancel.cpp new file mode 100644 index 0000000000..69dbff4bb4 --- /dev/null +++ b/indra/llcorehttp/_httpopcancel.cpp @@ -0,0 +1,82 @@ +/** + * @file _httpopcancel.cpp + * @brief Definitions for internal class HttpOpCancel + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpopcancel.h" + +#include +#include + +#include "httpcommon.h" +#include "httphandler.h" +#include "httpresponse.h" + +#include "_httprequestqueue.h" +#include "_httpreplyqueue.h" +#include "_httpservice.h" +#include "_httppolicy.h" +#include "_httplibcurl.h" + + +namespace LLCore +{ + + +// ================================== +// HttpOpCancel +// ================================== + + +HttpOpCancel::HttpOpCancel(HttpHandle handle) + : HttpOperation(), + mHandle(handle) +{} + + +HttpOpCancel::~HttpOpCancel() +{} + + +void HttpOpCancel::stageFromRequest(HttpService * service) +{ + // *FIXME: Need cancel functionality into services + addAsReply(); +} + + +void HttpOpCancel::visitNotifier(HttpRequest * request) +{ + if (mLibraryHandler) + { + HttpResponse * response = new HttpResponse(); + mLibraryHandler->onCompleted(static_cast(this), response); + response->release(); + } +} + + +} // end namespace LLCore + + diff --git a/indra/llcorehttp/_httpopcancel.h b/indra/llcorehttp/_httpopcancel.h new file mode 100644 index 0000000000..38ccc585ed --- /dev/null +++ b/indra/llcorehttp/_httpopcancel.h @@ -0,0 +1,75 @@ +/** + * @file _httpopcancel.h + * @brief Internal declarations for the HttpOpCancel subclass + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_OPCANCEL_H_ +#define _LLCORE_HTTP_OPCANCEL_H_ + + +#include "linden_common.h" // Modifies curl/curl.h interfaces + +#include "httpcommon.h" + +#include + +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// HttpOpCancel requests that a previously issued request +/// be canceled, if possible. Requests that have been made +/// active and are available for sending on the wire cannot +/// be canceled. + +class HttpOpCancel : public HttpOperation +{ +public: + HttpOpCancel(HttpHandle handle); + virtual ~HttpOpCancel(); + +private: + HttpOpCancel(const HttpOpCancel &); // Not defined + void operator=(const HttpOpCancel &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + + virtual void visitNotifier(HttpRequest * request); + +public: + // Request data + HttpHandle mHandle; + +}; // end class HttpOpCancel + + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_OPCANCEL_H_ + diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index d04961c47b..5d06a28586 100644 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -87,31 +87,6 @@ public: }; // end class HttpOperation -/// HttpOpCancel requests that a previously issued request -/// be canceled, if possible. Requests that have been made -/// active and are available for sending on the wire cannot -/// be canceled. - -class HttpOpCancel : public HttpOperation -{ -public: - HttpOpCancel(); - virtual ~HttpOpCancel(); - -private: - HttpOpCancel(const HttpOpCancel &); // Not defined - void operator=(const HttpOpCancel &); // Not defined - -public: - virtual void stageFromRequest(HttpService *); - virtual void stageFromReady(HttpService *); - virtual void stageFromActive(HttpService *); - -public: - HttpHandle mHandle; -}; // end class HttpOpCancel - - /// HttpOpStop requests the servicing thread to shutdown /// operations, cease pulling requests from the request /// queue and release shared resources (particularly diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 3c9eb71b9a..521bd5b879 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -86,53 +86,53 @@ HttpOpRequest::HttpOpRequest() mReqMethod(HOR_GET), mReqBody(NULL), mReqOffset(0), - mReqLen(0), + mReqLength(0), mReqHeaders(NULL), mReqOptions(NULL), mCurlActive(false), mCurlHandle(NULL), - mCurlHeaders(NULL), mCurlService(NULL), - mReplyStatus(200), + mCurlHeaders(NULL), mReplyBody(NULL), mReplyOffset(0), - mReplyLen(0), + mReplyLength(0), mReplyHeaders(NULL) {} HttpOpRequest::~HttpOpRequest() { - if (mCurlHandle) - { - curl_easy_cleanup(mCurlHandle); - mCurlHandle = NULL; - } - - if (mCurlHeaders) - { - curl_slist_free_all(mCurlHeaders); - mCurlHeaders = NULL; - } - - mCurlService = NULL; - if (mReqBody) { mReqBody->release(); mReqBody = NULL; } + if (mReqOptions) + { + mReqOptions->release(); + mReqOptions = NULL; + } + if (mReqHeaders) { - curl_slist_free_all(mReqHeaders); + mReqHeaders->release(); mReqHeaders = NULL; } - if (mReqOptions) + if (mCurlHandle) { - mReqOptions->release(); - mReqOptions = NULL; + curl_easy_cleanup(mCurlHandle); + mCurlHandle = NULL; + } + + mCurlService = NULL; + + + if (mCurlHeaders) + { + curl_slist_free_all(mCurlHeaders); + mCurlHeaders = NULL; } if (mReplyBody) @@ -165,28 +165,29 @@ void HttpOpRequest::stageFromReady(HttpService * service) void HttpOpRequest::stageFromActive(HttpService * service) { - if (mReplyLen) + if (mReplyLength) { // If non-zero, we received and processed a Content-Range // header with the response. Verify that what it says // is consistent with the received data. - if (mReplyLen != mReplyBody->size()) + if (mReplyLength != mReplyBody->size()) { // Not as expected, fail the request mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); } } - if (mReqHeaders) + if (mCurlHeaders) { // We take these headers out of the request now as they were // allocated originally in this thread and the notifier doesn't // need them. This eliminates one source of heap moving across // threads. - curl_slist_free_all(mReqHeaders); - mReqHeaders = NULL; + curl_slist_free_all(mCurlHeaders); + mCurlHeaders = NULL; } + addAsReply(); } @@ -196,12 +197,24 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) if (mLibraryHandler) { HttpResponse * response = new HttpResponse(); - - // *FIXME: add http status, offset, length response->setStatus(mStatus); - response->setReplyStatus(mReplyStatus); response->setBody(mReplyBody); response->setHeaders(mReplyHeaders); + unsigned int offset(0), length(0); + if (mReplyOffset || mReplyLength) + { + // Got an explicit offset/length in response + offset = mReplyOffset; + length = mReplyLength; + } + else if (mReplyBody) + { + // Provide implicit offset/length from request/response + offset = mReqOffset; + length = mReplyBody->size(); + } + response->setRange(offset, length); + mLibraryHandler->onCompleted(static_cast(this), response); response->release(); @@ -235,14 +248,15 @@ HttpStatus HttpOpRequest::setupGetByteRange(unsigned int policy_id, mReqMethod = HOR_GET; mReqURL = url; mReqOffset = offset; - mReqLen = len; + mReqLength = len; if (offset || len) { mProcFlags |= PF_SCAN_RANGE_HEADER; } if (headers && ! mReqHeaders) { - mReqHeaders = append_headers_to_slist(headers, mReqHeaders); + headers->addRef(); + mReqHeaders = headers; } if (options && ! mReqOptions) { @@ -257,7 +271,7 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service) { // *FIXME: better error handling later HttpStatus status; - + mCurlHandle = curl_easy_init(); curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30); curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, 30); @@ -268,14 +282,18 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); // curl_easy_setopt(handle, CURLOPT_PROXY, ""); - mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); - curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle); - if (mReqOffset || mReqLen) + if (mReqHeaders) + { + mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); + } + mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); + + if (mReqOffset || mReqLength) { static const char * fmt1("Range: bytes=%d-%d"); static const char * fmt2("Range: bytes=%d-"); @@ -284,16 +302,17 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service) #if defined(WIN32) _snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1, - (mReqLen ? fmt1 : fmt2), - mReqOffset, mReqOffset + mReqLen - 1); + (mReqLength ? fmt1 : fmt2), + mReqOffset, mReqOffset + mReqLength - 1); #else snprintf(range_line, sizeof(range_line), - (mReqLen ? fmt1 : fmt2), - mReqOffset, mReqOffset + mReqLen - 1); + (mReqLength ? fmt1 : fmt2), + mReqOffset, mReqOffset + mReqLength - 1); #endif // defined(WIN32) range_line[sizeof(range_line) - 1] = '\0'; mCurlHeaders = curl_slist_append(mCurlHeaders, range_line); } + curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) { @@ -347,8 +366,9 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi if (hdr_size >= status_line_len && ! strncmp(status_line, hdr_data, status_line_len)) { // One of possibly several status lines. Reset what we know and start over + // taking results from the last header stanza we receive. op->mReplyOffset = 0; - op->mReplyLen = 0; + op->mReplyLength = 0; op->mStatus = HttpStatus(); } else if (op->mProcFlags & PF_SCAN_RANGE_HEADER) @@ -371,7 +391,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi { // Success, record the fragment position op->mReplyOffset = first; - op->mReplyLen = last - first + 1; + op->mReplyLength = last - first + 1; } else if (-1 == status) { @@ -390,6 +410,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi if (op->mProcFlags & PF_SAVE_HEADERS) { // Save headers in response + // *FIXME: Implement this... ; } diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 232ee841d6..601937a943 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -103,22 +103,21 @@ public: std::string mReqURL; BufferArray * mReqBody; off_t mReqOffset; - size_t mReqLen; - curl_slist * mReqHeaders; + size_t mReqLength; + HttpHeaders * mReqHeaders; HttpOptions * mReqOptions; // Transport data bool mCurlActive; CURL * mCurlHandle; - curl_slist * mCurlHeaders; HttpService * mCurlService; + curl_slist * mCurlHeaders; // Result data HttpStatus mStatus; - int mReplyStatus; BufferArray * mReplyBody; off_t mReplyOffset; - size_t mReplyLen; + size_t mReplyLength; HttpHeaders * mReplyHeaders; }; // end class HttpOpRequest diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 28aea27f38..192bc73b31 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -39,6 +39,7 @@ class HttpService; class HttpOpRequest; +/// Implements class-based queuing policies for an HttpService instance. class HttpPolicy { public: @@ -58,7 +59,7 @@ protected: typedef std::vector ready_queue_t; protected: - HttpService * mService; // Naked pointer, not refcounted + HttpService * mService; // Naked pointer, not refcounted, not owner ready_queue_t mReadyQueue; }; // end class HttpPolicy diff --git a/indra/llcorehttp/_httpreplyqueue.h b/indra/llcorehttp/_httpreplyqueue.h index 56dadec87c..28cb1d68b7 100644 --- a/indra/llcorehttp/_httpreplyqueue.h +++ b/indra/llcorehttp/_httpreplyqueue.h @@ -58,7 +58,7 @@ class HttpOperation; /// will be coded anyway so it shouldn't be too much of a /// burden. -class HttpReplyQueue: public LLCoreInt::RefCounted +class HttpReplyQueue : public LLCoreInt::RefCounted { public: /// Caller acquires a Refcount on construction diff --git a/indra/llcorehttp/_httprequestqueue.h b/indra/llcorehttp/_httprequestqueue.h index 3a9ce0c3c6..f96bd7520c 100644 --- a/indra/llcorehttp/_httprequestqueue.h +++ b/indra/llcorehttp/_httprequestqueue.h @@ -46,7 +46,7 @@ class HttpOperation; /// requests from all HttpRequest instances into the /// singleton HttpService instance. -class HttpRequestQueue: public LLCoreInt::RefCounted +class HttpRequestQueue : public LLCoreInt::RefCounted { protected: /// Caller acquires a Refcount on construction diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index c052e35452..ba76e1eeca 100644 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -152,8 +152,8 @@ protected: LLCoreInt::HttpThread * mThread; // === working-thread-only data === - HttpPolicy * mPolicy; - HttpLibcurl * mTransport; + HttpPolicy * mPolicy; // Simple pointer, has ownership + HttpLibcurl * mTransport; // Simple pointer, has ownership }; // end class HttpService diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index c37d081150..b5872606b8 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -37,22 +37,82 @@ HttpStatus::type_enum_t EXT_CURL_EASY; HttpStatus::type_enum_t EXT_CURL_MULTI; HttpStatus::type_enum_t LLCORE; +HttpStatus::operator unsigned long() const +{ + static const int shift(sizeof(unsigned long) * 4); + + unsigned long result(((unsigned long) mType) << shift | (unsigned long) (int) mStatus); + return result; +} + + std::string HttpStatus::toString() const { static const char * llcore_errors[] = { "", + "HTTP error reply status", "Services shutting down", "Operation canceled", "Invalid Content-Range header encountered" }; static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); + + static const struct + { + type_enum_t mCode; + char * mText; + } + http_errors[] = + { + // Keep sorted by mCode, we binary search this list. + { 100, "Continue" }, + { 101, "Switching Protocols" }, + { 200, "OK" }, + { 201, "Created" }, + { 202, "Accepted" }, + { 203, "Non-Authoritative Information" }, + { 204, "No Content" }, + { 205, "Reset Content" }, + { 206, "Partial Content" }, + { 300, "Multiple Choices" }, + { 301, "Moved Permanently" }, + { 302, "Found" }, + { 303, "See Other" }, + { 304, "Not Modified" }, + { 305, "Use Proxy" }, + { 307, "Temporary Redirect" }, + { 400, "Bad Request" }, + { 401, "Unauthorized" }, + { 402, "Payment Required" }, + { 403, "Forbidden" }, + { 404, "Not Found" }, + { 405, "Method Not Allowed" }, + { 406, "Not Acceptable" }, + { 407, "Proxy Authentication Required" }, + { 408, "Request Time-out" }, + { 409, "Conflict" }, + { 410, "Gone" }, + { 411, "Length Required" }, + { 412, "Precondition Failed" }, + { 413, "Request Entity Too Large" }, + { 414, "Request-URI Too Large" }, + { 415, "Unsupported Media Type" }, + { 416, "Requested range not satisfiable" }, + { 417, "Expectation Failed" }, + { 500, "Internal Server Error" }, + { 501, "Not Implemented" }, + { 502, "Bad Gateway" }, + { 503, "Service Unavailable" }, + { 504, "Gateway Time-out" }, + { 505, "HTTP Version not supported" } + }; + static const int http_errors_count(sizeof(http_errors) / sizeof(http_errors[0])); if (*this) { return std::string(""); } - switch (mType) { case EXT_CURL_EASY: @@ -67,7 +127,33 @@ std::string HttpStatus::toString() const return std::string(llcore_errors[mStatus]); } break; - + + default: + if (isHttpStatus()) + { + int bottom(0), top(http_errors_count); + while (true) + { + int at((bottom + top) / 2); + if (mType == http_errors[at].mCode) + { + return std::string(http_errors[at].mText); + } + if (at == bottom) + { + break; + } + else if (mType < http_errors[at].mCode) + { + top = at; + } + else + { + bottom = at; + } + } + } + break; } return std::string("Unknown error"); } diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index 9de5769d57..f81be7103e 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -122,23 +122,38 @@ enum HttpError // Successful value compatible with the libcurl codes. HE_SUCCESS = 0, + // Intended for HTTP reply codes 100-999, indicates that + // the reply should be considered an error by the application. + HE_REPLY_ERROR = 1, + // Service is shutting down and requested operation will // not be queued or performed. - HE_SHUTTING_DOWN = 1, + HE_SHUTTING_DOWN = 2, // Operation was canceled by request. - HE_OP_CANCELED = 2, + HE_OP_CANCELED = 3, // Invalid content range header received. - HE_INV_CONTENT_RANGE_HDR = 3 + HE_INV_CONTENT_RANGE_HDR = 4 }; // end enum HttpError -/// HttpStatus encapsulates errors from libcurl (easy, multi) as well as -/// internal errors. The encapsulation isn't expected to completely -/// isolate the caller from libcurl but basic operational tests (success -/// or failure) are provided. +/// HttpStatus encapsulates errors from libcurl (easy, multi), HTTP +/// reply status codes and internal errors as well. The encapsulation +/// isn't expected to completely isolate the caller from libcurl but +/// basic operational tests (success or failure) are provided. +/// +/// Non-HTTP status are encoded as (type, status) with type being +/// one of: EXT_CURL_EASY, EXT_CURL_MULTI or LLCORE and status +/// being the success/error code from that domain. HTTP status +/// is encoded as (status, error_flag). Status should be in the +/// range [100, 999] and error_flag is either HE_SUCCESS or +/// HE_REPLY_ERROR to indicate whether this should be treated as +/// a successful status or an error. The application is responsible +/// for making that determination and a range like [200, 299] isn't +/// automatically assumed to be definitive. + struct HttpStatus { typedef unsigned short type_enum_t; @@ -192,12 +207,42 @@ struct HttpStatus return 0 != mStatus; } + /// Equality and inequality tests to bypass bool conversion + /// which will do the wrong thing in conditional expressions. + bool operator==(const HttpStatus & rhs) const + { + return mType == rhs.mType && mStatus == rhs.mStatus; + } + + bool operator!=(const HttpStatus & rhs) const + { + return ! operator==(rhs); + } + + /// Convert to single numeric representation. Mainly + /// for logging or other informal purposes. Also + /// creates an ambiguous second path to integer conversion + /// which tends to find programming errors such as formatting + /// the status to a stream (operator<<). + operator unsigned long() const; + unsigned long toULong() const + { + return operator unsigned long(); + } + /// Convert status to a string representation. For /// success, returns an empty string. For failure /// statuses, a string as appropriate for the source of /// the error code (libcurl easy, libcurl multi, or /// LLCore itself). std::string toString() const; + + /// Returns true if the status value represents an + /// HTTP response status (100 - 999). + bool isHttpStatus() const + { + return mType >= type_enum_t(100) && mType <= type_enum_t(999); + } }; // end struct HttpStatus diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 2a87f5231a..6c62f931ff 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -31,6 +31,7 @@ #include "_httpservice.h" #include "_httpoperation.h" #include "_httpoprequest.h" +#include "_httpopcancel.h" namespace @@ -189,6 +190,22 @@ HttpHandle HttpRequest::requestGetByteRange(unsigned int policy_id, } +HttpHandle HttpRequest::requestCancel(HttpHandle handle, HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpCancel * op = new HttpOpCancel(handle); + op->setHandlers(mReplyQueue, mSelfHandler, user_handler); + mRequestQueue->addOp(op); // transfer refcount as well + + mLastReqStatus = status; + ret_handle = static_cast(op); + + return ret_handle; +} + + HttpHandle HttpRequest::requestNoOp(HttpHandler * user_handler) { HttpStatus status; diff --git a/indra/llcorehttp/httpresponse.cpp b/indra/llcorehttp/httpresponse.cpp index 9ac8276f05..3dcdadb337 100644 --- a/indra/llcorehttp/httpresponse.cpp +++ b/indra/llcorehttp/httpresponse.cpp @@ -35,7 +35,8 @@ namespace LLCore HttpResponse::HttpResponse() : LLCoreInt::RefCounted(true), - mReplyStatus(0U), + mReplyOffset(0U), + mReplyLength(0U), mBufferArray(NULL), mHeaders(NULL) {} diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index a25e22aef6..5cf3a919f4 100644 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -67,8 +67,6 @@ protected: public: /// Returns the final status of the requested operation. /// - // *FIXME: Haven't incorporated HTTP status into this yet. - // Will do soon. HttpStatus getStatus() const { return mStatus; @@ -79,19 +77,6 @@ public: mStatus = status; } - /// Fetch the HTTP reply status. This is only guaranteed to be - /// valid if the HttpStatus tests successful and was the result - /// of a completed HTTP request. - unsigned int getReplyStatus() const - { - return mReplyStatus; - } - - void setReplyStatus(unsigned int status) - { - mReplyStatus = status; - } - /// Simple getter for the response body returned as a scatter/gather /// buffer. If the operation doesn't produce data (such as the Null /// or StopThread operations), this may be NULL. @@ -122,11 +107,26 @@ public: /// Behaves like @see setResponse() but for header data. void setHeaders(HttpHeaders * headers); + + /// If a 'Range:' header was used, these methods are involved + /// in setting and returning data about the actual response. + void getRange(unsigned int * offset, unsigned int * length) const + { + *offset = mReplyOffset; + *length = mReplyLength; + } + + void setRange(unsigned int offset, unsigned int length) + { + mReplyOffset = offset; + mReplyLength = length; + } protected: // Response data here HttpStatus mStatus; - unsigned int mReplyStatus; + unsigned int mReplyOffset; + unsigned int mReplyLength; BufferArray * mBufferArray; HttpHeaders * mHeaders; }; diff --git a/indra/llcorehttp/tests/test_httpstatus.hpp b/indra/llcorehttp/tests/test_httpstatus.hpp index 38bf494dec..f7b542d3b5 100644 --- a/indra/llcorehttp/tests/test_httpstatus.hpp +++ b/indra/llcorehttp/tests/test_httpstatus.hpp @@ -157,6 +157,108 @@ void HttpStatusTestObjectType::test<4>() ensure(! msg.empty()); } +template <> template <> +void HttpStatusTestObjectType::test<5>() +{ + set_test_name("HttpStatus equality/inequality testing"); + + // Make certain equality/inequality tests do not pass + // through the bool conversion. Distinct successful + // and error statuses should compare unequal. + + HttpStatus status1(HttpStatus::LLCORE, HE_SUCCESS); + HttpStatus status2(HttpStatus::EXT_CURL_EASY, HE_SUCCESS); + ensure(status1 != status2); + + status1.mType = HttpStatus::LLCORE; + status1.mStatus = HE_REPLY_ERROR; + status2.mType = HttpStatus::LLCORE; + status2.mStatus= HE_SHUTTING_DOWN; + ensure(status1 != status2); +} + +template <> template <> +void HttpStatusTestObjectType::test<6>() +{ + set_test_name("HttpStatus basic HTTP status encoding"); + + HttpStatus status; + status.mType = 200; + status.mStatus = HE_SUCCESS; + std::string msg = status.toString(); + ensure(msg.empty()); + ensure(bool(status)); + + // Normally a success but application says error + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(! msg.empty()); + ensure(! bool(status)); + ensure(status.toULong() > 1UL); // Biggish number, not a bool-to-ulong + + // Same statuses with distinct success/fail are distinct + status.mType = 200; + status.mStatus = HE_SUCCESS; + HttpStatus status2(200, HE_REPLY_ERROR); + ensure(status != status2); + + // Normally an error but application says okay + status.mType = 406; + status.mStatus = HE_SUCCESS; + msg = status.toString(); + ensure(msg.empty()); + ensure(bool(status)); + + // Different statuses but both successful are distinct + status.mType = 200; + status.mStatus = HE_SUCCESS; + status2.mType = 201; + status2.mStatus = HE_SUCCESS; + ensure(status != status2); + + // Different statuses but both failed are distinct + status.mType = 200; + status.mStatus = HE_REPLY_ERROR; + status2.mType = 201; + status2.mStatus = HE_REPLY_ERROR; + ensure(status != status2); +} + +template <> template <> +void HttpStatusTestObjectType::test<7>() +{ + set_test_name("HttpStatus HTTP error text strings"); + + HttpStatus status(100, HE_REPLY_ERROR); + std::string msg(status.toString()); + ensure(! msg.empty()); // Should be something + ensure(msg == "Continue"); + + status.mStatus = HE_SUCCESS; + msg = status.toString(); + ensure(msg.empty()); // Success is empty + + status.mType = 199; + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "Unknown error"); + + status.mType = 505; // Last defined string + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "HTTP Version not supported"); + + status.mType = 506; // One beyond + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "Unknown error"); + + status.mType = 999; // Last HTTP status + status.mStatus = HE_REPLY_ERROR; + msg = status.toString(); + ensure(msg == "Unknown error"); +} + } // end namespace tut #endif // TEST_HTTP_STATUS_H diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 1174d108d2..8e6deb9cce 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -212,6 +212,11 @@ #include "llmachineid.h" #include "llmainlooprepeater.h" +// LLCore::HTTP +#include "httpcommon.h" +#include "httprequest.h" +#include "httphandler.h" + // *FIX: These extern globals should be cleaned up. // The globals either represent state/config/resource-storage of either // this app, or another 'component' of the viewer. App globals should be @@ -326,6 +331,53 @@ static std::string gLaunchFileOnQuit; // Used on Win32 for other apps to identify our window (eg, win_setup) const char* const VIEWER_WINDOW_CLASSNAME = "Second Life"; +namespace +{ + +// This class manages the lifecyle of the core http library. +// Slightly different style than traditional code but reflects +// the use of handler classes and light-weight interface +// object instances of the new libraries. To be used +// as a singleton and static construction is fine. +class CoreHttp : public LLCore::HttpHandler +{ +public: + CoreHttp(); + ~CoreHttp(); + + // Initialize the LLCore::HTTP library creating service classes + // and starting the servicing thread. Caller is expected to do + // other initializations (SSL mutex, thread hash function) appropriate + // for the application. + void init(); + + // Request that the servicing thread stop servicing requests, + // release resource references and stop. + void requestStop(); + + // Terminate LLCore::HTTP library services. Caller is expected + // to have made a best-effort to shutdown the servicing thread + // by issuing a requestThreadStop() and waiting for completion + // notification that the stop has completed. + void cleanup(); + + // Notification when the stop request is complete. + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); + +private: + static const F64 MAX_THREAD_WAIT_TIME; + +private: + LLCore::HttpRequest * mRequest; + LLCore::HttpHandle mStopHandle; + F64 mStopRequested; + bool mStopped; +}; + +CoreHttp coreHttpLib; + +} // end anonymous namespace + //-- LLDeferredTaskList ------------------------------------------------------ /** @@ -720,6 +772,9 @@ bool LLAppViewer::init() LLViewerStatsRecorder::initClass(); #endif + // Initialize the non-LLCurl libcurl library + coreHttpLib.init(); + // *NOTE:Mani - LLCurl::initClass is not thread safe. // Called before threads are created. LLCurl::initClass(gSavedSettings.getF32("CurlRequestTimeOut"), @@ -1807,6 +1862,7 @@ bool LLAppViewer::cleanup() // Delete workers first // shotdown all worker threads before deleting them in case of co-dependencies + coreHttpLib.requestStop(); sTextureFetch->shutdown(); sTextureCache->shutdown(); sImageDecodeThread->shutdown(); @@ -1890,6 +1946,9 @@ bool LLAppViewer::cleanup() // *NOTE:Mani - The following call is not thread safe. LLCurl::cleanupClass(); + // Non-LLCurl libcurl library + coreHttpLib.cleanup(); + // If we're exiting to launch an URL, do that here so the screen // is at the right resolution before we launch IE. if (!gLaunchFileOnQuit.empty()) @@ -5267,3 +5326,103 @@ void LLAppViewer::metricsSend(bool enable_reporting) gViewerAssetStatsMain->reset(); } +namespace +{ + +const F64 CoreHttp::MAX_THREAD_WAIT_TIME(10.0); + +CoreHttp::CoreHttp() + : mRequest(NULL), + mStopHandle(LLCORE_HTTP_HANDLE_INVALID), + mStopRequested(0.0), + mStopped(false) +{} + + +CoreHttp::~CoreHttp() +{ + delete mRequest; + mRequest = NULL; +} + + +void CoreHttp::init() +{ + LLCore::HttpStatus status = LLCore::HttpRequest::createService(); + if (! status) + { + LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " + << status.toString() + << LL_ENDL; + } + + status = LLCore::HttpRequest::startThread(); + if (! status) + { + LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " + << status.toString() + << LL_ENDL; + } + + mRequest = new LLCore::HttpRequest; +} + + +void CoreHttp::requestStop() +{ + llassert_always(mRequest); + + mStopHandle = mRequest->requestStopThread(this); + if (LLCORE_HTTP_HANDLE_INVALID != mStopHandle) + { + mStopRequested = LLTimer::getTotalSeconds(); + } +} + + +void CoreHttp::cleanup() +{ + if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) + { + // Should have been started already... + requestStop(); + } + + if (LLCORE_HTTP_HANDLE_INVALID == mStopHandle) + { + LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services without thread shutdown" + << LL_ENDL; + } + else + { + while (! mStopped && LLTimer::getTotalSeconds() < (mStopRequested + MAX_THREAD_WAIT_TIME)) + { + mRequest->update(200); + ms_sleep(50); + } + if (! mStopped) + { + LL_WARNS("Cleanup") << "Attempting to cleanup HTTP services with thread shutdown incomplete" + << LL_ENDL; + } + } + + delete mRequest; + mRequest = NULL; + + LLCore::HttpStatus status = LLCore::HttpRequest::destroyService(); + if (! status) + { + LL_WARNS("Cleanup") << "Failed to shutdown HTTP services, continuing. Reason: " + << status.toString() + << LL_ENDL; + } +} + + +void CoreHttp::onCompleted(LLCore::HttpHandle, LLCore::HttpResponse *) +{ + mStopped = true; +} + +} // end anonymous namespace diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index f18aa8b4e6..17c68f7c22 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2000&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2012, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -33,7 +33,6 @@ #include "lltexturefetch.h" -#include "llcurl.h" #include "lldir.h" #include "llhttpclient.h" #include "llhttpstatuscodes.h" @@ -53,11 +52,17 @@ #include "llviewerassetstats.h" #include "llworld.h" +#include "httprequest.h" +#include "httphandler.h" +#include "httpresponse.h" +#include "bufferarray.h" + + ////////////////////////////////////////////////////////////////////////////// -class LLTextureFetchWorker : public LLWorkerClass +class LLTextureFetchWorker : public LLWorkerClass, public LLCore::HttpHandler + { friend class LLTextureFetch; - friend class HTTPGetResponder; private: class CacheReadResponder : public LLTextureCache::ReadResponder @@ -147,15 +152,14 @@ public: ~LLTextureFetchWorker(); // void relese() { --mActiveCount; } - S32 callbackHttpGet(const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer, - bool partial, bool success); + S32 callbackHttpGet(LLCore::HttpResponse * response, + bool partial, bool success); void callbackCacheRead(bool success, LLImageFormatted* image, S32 imagesize, BOOL islocal); void callbackCacheWrite(bool success); void callbackDecoded(bool success, LLImageRaw* raw, LLImageRaw* aux); - void setGetStatus(U32 status, const std::string& reason) + void setGetStatus(LLCore::HttpStatus status, const std::string& reason) { LLMutexLock lock(&mWorkMutex); @@ -167,6 +171,9 @@ public: bool getCanUseHTTP() const { return mCanUseHTTP; } LLTextureFetch & getFetcher() { return *mFetcher; } + + // Inherited from LLCore::HttpHandler + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); protected: LLTextureFetchWorker(LLTextureFetch* fetcher, const std::string& url, const LLUUID& id, const LLHost& host, @@ -191,9 +198,15 @@ private: void lockWorkMutex() { mWorkMutex.lock(); } void unlockWorkMutex() { mWorkMutex.unlock(); } + void recordTextureStart(bool is_http); + void recordTextureDone(bool is_http); + private: enum e_state // mState { + // *NOTE: Do not change the order/value of state variables, some code + // depends upon specific ordering/adjacency. + // NOTE: Affects LLTextureBar::draw in lltextureview.cpp (debug hack) INVALID = 0, INIT, @@ -244,9 +257,8 @@ private: LLFrameTimer mFetchTimer; LLTextureCache::handle_t mCacheReadHandle; LLTextureCache::handle_t mCacheWriteHandle; - U8* mBuffer; - S32 mBufferSize; S32 mRequestedSize; + S32 mRequestedOffset; S32 mDesiredSize; S32 mFileSize; S32 mCachedSize; @@ -263,7 +275,7 @@ private: S32 mHTTPFailCount; S32 mRetryAttempt; S32 mActiveCount; - U32 mGetStatus; + LLCore::HttpStatus mGetStatus; std::string mGetReason; // Work Data @@ -283,106 +295,17 @@ private: U8 mImageCodec; LLViewerAssetStats::duration_t mMetricsStartTime; + + LLCore::HttpHandle mHttpHandle; + LLCore::BufferArray * mHttpBufferArray; + int mHttpPolicyClass; + bool mHttpActive; + unsigned int mHttpReplySize; + unsigned int mHttpReplyOffset; }; ////////////////////////////////////////////////////////////////////////////// -class HTTPGetResponder : public LLCurl::Responder -{ - LOG_CLASS(HTTPGetResponder); -public: - HTTPGetResponder(LLTextureFetch* fetcher, const LLUUID& id, U64 startTime, S32 requestedSize, U32 offset, bool redir) - : mFetcher(fetcher), mID(id), mStartTime(startTime), mRequestedSize(requestedSize), mOffset(offset), mFollowRedir(redir) - { - } - ~HTTPGetResponder() - { - } - - virtual void completedRaw(U32 status, const std::string& reason, - const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer) - { - static LLCachedControl log_to_viewer_log(gSavedSettings,"LogTextureDownloadsToViewerLog"); - static LLCachedControl log_to_sim(gSavedSettings,"LogTextureDownloadsToSimulator"); - static LLCachedControl log_texture_traffic(gSavedSettings,"LogTextureNetworkTraffic") ; - - if (log_to_viewer_log || log_to_sim) - { - mFetcher->mTextureInfo.setRequestStartTime(mID, mStartTime); - U64 timeNow = LLTimer::getTotalTime(); - mFetcher->mTextureInfo.setRequestType(mID, LLTextureInfoDetails::REQUEST_TYPE_HTTP); - mFetcher->mTextureInfo.setRequestSize(mID, mRequestedSize); - mFetcher->mTextureInfo.setRequestOffset(mID, mOffset); - mFetcher->mTextureInfo.setRequestCompleteTimeAndLog(mID, timeNow); - } - - lldebugs << "HTTP COMPLETE: " << mID << llendl; - LLTextureFetchWorker* worker = mFetcher->getWorker(mID); - if (worker) - { - bool success = false; - bool partial = false; - if (HTTP_OK <= status && status < HTTP_MULTIPLE_CHOICES) - { - success = true; - if (HTTP_PARTIAL_CONTENT == status) // partial information - { - partial = true; - } - } - - if (!success) - { - worker->setGetStatus(status, reason); -// llwarns << "CURL GET FAILED, status:" << status << " reason:" << reason << llendl; - } - - S32 data_size = worker->callbackHttpGet(channels, buffer, partial, success); - - if(log_texture_traffic && data_size > 0) - { - LLViewerTexture* tex = LLViewerTextureManager::findTexture(mID) ; - if(tex) - { - gTotalTextureBytesPerBoostLevel[tex->getBoostLevel()] += data_size ; - } - } - - mFetcher->removeFromHTTPQueue(mID, data_size); - - if (worker->mMetricsStartTime) - { - LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE, - true, - LLImageBase::TYPE_AVATAR_BAKE == worker->mType, - LLViewerAssetStatsFF::get_timestamp() - worker->mMetricsStartTime); - worker->mMetricsStartTime = 0; - } - LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE, - true, - LLImageBase::TYPE_AVATAR_BAKE == worker->mType); - } - else - { - mFetcher->removeFromHTTPQueue(mID); - llwarns << "Worker not found: " << mID << llendl; - } - } - - virtual bool followRedir() - { - return mFollowRedir; - } - -private: - LLTextureFetch* mFetcher; - LLUUID mID; - U64 mStartTime; - S32 mRequestedSize; - U32 mOffset; - bool mFollowRedir; -}; ////////////////////////////////////////////////////////////////////////////// @@ -639,6 +562,7 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, S32 discard, // Desired discard S32 size) // Desired size : LLWorkerClass(fetcher, "TextureFetch"), + LLCore::HttpHandler(), mState(INIT), mWriteToCacheState(NOT_WRITE), mFetcher(fetcher), @@ -655,9 +579,8 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mDecodedDiscard(-1), mCacheReadHandle(LLTextureCache::nullHandle()), mCacheWriteHandle(LLTextureCache::nullHandle()), - mBuffer(NULL), - mBufferSize(0), mRequestedSize(0), + mRequestedOffset(0), mDesiredSize(TEXTURE_CACHE_ENTRY_SIZE), mFileSize(0), mCachedSize(0), @@ -673,13 +596,18 @@ LLTextureFetchWorker::LLTextureFetchWorker(LLTextureFetch* fetcher, mHTTPFailCount(0), mRetryAttempt(0), mActiveCount(0), - mGetStatus(0), mWorkMutex(NULL), mFirstPacket(0), mLastPacket(-1), mTotalPackets(0), mImageCodec(IMG_CODEC_INVALID), - mMetricsStartTime(0) + mMetricsStartTime(0), + mHttpHandle(LLCORE_HTTP_HANDLE_INVALID), + mHttpBufferArray(NULL), + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpActive(false), + mHttpReplySize(0U), + mHttpReplyOffset(0U) { mCanUseNET = mUrl.empty() ; @@ -701,6 +629,11 @@ LLTextureFetchWorker::~LLTextureFetchWorker() // << " Requested=" << mRequestedDiscard // << " Desired=" << mDesiredDiscard << llendl; llassert_always(!haveWork()); + if (mHttpActive) + { + LL_WARNS("Texture") << "Deleting worker object while HTTP request is active." + << LL_ENDL; + } lockWorkMutex(); if (mCacheReadHandle != LLTextureCache::nullHandle() && mFetcher->mTextureCache) { @@ -714,6 +647,11 @@ LLTextureFetchWorker::~LLTextureFetchWorker() clearPackets(); unlockWorkMutex(); mFetcher->removeFromHTTPQueue(mID); + if (mHttpBufferArray) + { + mHttpBufferArray->release(); + mHttpBufferArray = NULL; + } } void LLTextureFetchWorker::clearPackets() @@ -797,7 +735,7 @@ void LLTextureFetchWorker::setDesiredDiscard(S32 discard, S32 size) if ((prioritize && mState == INIT) || mState == DONE) { mState = INIT; - U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_HIGH; + U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_LOW; setPriority(work_priority); } } @@ -810,16 +748,18 @@ void LLTextureFetchWorker::setImagePriority(F32 priority) { mImagePriority = priority; calcWorkPriority(); - U32 work_priority = mWorkPriority | (getPriority() & LLWorkerThread::PRIORITY_HIGHBITS); + U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_LOW; setPriority(work_priority); } } void LLTextureFetchWorker::resetFormattedData() { - FREE_MEM(LLImageBase::getPrivatePool(), mBuffer); - mBuffer = NULL; - mBufferSize = 0; + if (mHttpBufferArray) + { + mHttpBufferArray->release(); + mHttpBufferArray = NULL; + } if (mFormattedImage.notNull()) { mFormattedImage->deleteData(); @@ -875,6 +815,14 @@ bool LLTextureFetchWorker::doWork(S32 param) mFetchTimer.reset(); } + static LLUUID last_id; + if (mID != last_id) + { + // LL_WARNS("Texture") << "DOWORK SWITCH: " << last_id << " to: " << mID + // << LL_ENDL; + last_id = mID; + } + if (mState == INIT) { mRawImage = NULL ; @@ -882,15 +830,18 @@ bool LLTextureFetchWorker::doWork(S32 param) mLoadedDiscard = -1; mDecodedDiscard = -1; mRequestedSize = 0; + mRequestedOffset = 0; mFileSize = 0; mCachedSize = 0; mLoaded = FALSE; mSentRequest = UNSENT; mDecoded = FALSE; mWritten = FALSE; - FREE_MEM(LLImageBase::getPrivatePool(), mBuffer); - mBuffer = NULL; - mBufferSize = 0; + if (mHttpBufferArray) + { + mHttpBufferArray->release(); + mHttpBufferArray = NULL; + } mHaveAllData = FALSE; clearPackets(); // TODO: Shouldn't be necessary mCacheReadHandle = LLTextureCache::nullHandle(); @@ -904,6 +855,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == LOAD_FROM_TEXTURE_CACHE) { + setPriority(0); // Set priority first since Responder may change it if (mCacheReadHandle == LLTextureCache::nullHandle()) { U32 cache_priority = mWorkPriority; @@ -919,8 +871,6 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mUrl.compare(0, 7, "file://") == 0) { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it - // read file from local disk std::string filename = mUrl.substr(7, std::string::npos); CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); @@ -929,8 +879,6 @@ bool LLTextureFetchWorker::doWork(S32 param) } else if (mUrl.empty()) { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it - CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); mCacheReadHandle = mFetcher->mTextureCache->readFromCache(mID, cache_priority, offset, size, responder); @@ -942,12 +890,12 @@ bool LLTextureFetchWorker::doWork(S32 param) // *TODO:?remove this warning llwarns << "Unknown URL Type: " << mUrl << llendl; } - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = SEND_HTTP_REQ; } else { - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = LOAD_FROM_NETWORK; } } @@ -959,6 +907,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { mCacheReadHandle = LLTextureCache::nullHandle(); mState = CACHE_POST; + setPriority(LLWorkerThread::PRIORITY_HIGH); // fall through } else @@ -982,6 +931,7 @@ bool LLTextureFetchWorker::doWork(S32 param) llassert_always(mFormattedImage->getDataSize() > 0); mLoadedDiscard = mDesiredDiscard; mState = DECODE_IMAGE; + setPriority(LLWorkerThread::PRIORITY_HIGH); mWriteToCacheState = NOT_WRITE ; LL_DEBUGS("Texture") << mID << ": Cached. Bytes: " << mFormattedImage->getDataSize() << " Size: " << llformat("%dx%d",mFormattedImage->getWidth(),mFormattedImage->getHeight()) @@ -999,6 +949,7 @@ bool LLTextureFetchWorker::doWork(S32 param) else { LL_DEBUGS("Texture") << mID << ": Not in Cache" << LL_ENDL; + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = LOAD_FROM_NETWORK; } // fall through @@ -1009,6 +960,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { static LLCachedControl use_http(gSavedSettings,"ImagePipelineUseHTTP"); + setPriority(0); // if (mHost != LLHost::invalid) get_url = false; if ( use_http && mCanUseHTTP && mUrl.empty())//get http url. { @@ -1040,8 +992,8 @@ bool LLTextureFetchWorker::doWork(S32 param) } if (mCanUseHTTP && !mUrl.empty()) { - mState = LLTextureFetchWorker::SEND_HTTP_REQ; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + mState = SEND_HTTP_REQ; + setPriority(LLWorkerThread::PRIORITY_HIGH); if(mWriteToCacheState != NOT_WRITE) { mWriteToCacheState = CAN_WRITE ; @@ -1057,14 +1009,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mRequestedDiscard = mDesiredDiscard; mSentRequest = QUEUED; mFetcher->addToNetworkQueue(this); - if (! mMetricsStartTime) - { - mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); - } - LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, - false, - LLImageBase::TYPE_AVATAR_BAKE == mType); - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + recordTextureStart(false); return false; } @@ -1074,12 +1019,7 @@ bool LLTextureFetchWorker::doWork(S32 param) //llassert_always(mFetcher->mNetworkQueue.find(mID) != mFetcher->mNetworkQueue.end()); // Make certain this is in the network queue //mFetcher->addToNetworkQueue(this); - //if (! mMetricsStartTime) - //{ - // mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); - //} - //LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, false, - // LLImageBase::TYPE_AVATAR_BAKE == mType); + //recordTextureStart(false); //setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; } @@ -1087,6 +1027,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == LOAD_FROM_SIMULATOR) { + setPriority(0); if (mFormattedImage.isNull()) { mFormattedImage = new LLImageJ2C; @@ -1101,39 +1042,22 @@ bool LLTextureFetchWorker::doWork(S32 param) // llwarns << "processSimulatorPackets() failed to load buffer" << llendl; return true; // failed } - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; mWriteToCacheState = SHOULD_WRITE; - - if (mMetricsStartTime) - { - LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE, - false, - LLImageBase::TYPE_AVATAR_BAKE == mType, - LLViewerAssetStatsFF::get_timestamp() - mMetricsStartTime); - mMetricsStartTime = 0; - } - LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE, - false, - LLImageBase::TYPE_AVATAR_BAKE == mType); + recordTextureDone(false); } else { mFetcher->addToNetworkQueue(this); // failsafe - if (! mMetricsStartTime) - { - mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); - } - LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, - false, - LLImageBase::TYPE_AVATAR_BAKE == mType); - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + recordTextureStart(false); } return false; } if (mState == SEND_HTTP_REQ) { + setPriority(0); if(mCanUseHTTP) { //NOTE: @@ -1159,6 +1083,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { // We already have all the data, just decode it mLoadedDiscard = mFormattedImage->getDiscardLevel(); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; return false; } @@ -1171,44 +1096,48 @@ bool LLTextureFetchWorker::doWork(S32 param) mRequestedSize = mDesiredSize; mRequestedDiscard = mDesiredDiscard; mRequestedSize -= cur_size; - S32 offset = cur_size; - mBufferSize = cur_size; // This will get modified by callbackHttpGet() + mRequestedOffset = cur_size; - bool res = false; + mHttpHandle = LLCORE_HTTP_HANDLE_INVALID; if (!mUrl.empty()) { mLoaded = FALSE; - mGetStatus = 0; + mGetStatus = LLCore::HttpStatus(); mGetReason.clear(); - LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << offset + LL_DEBUGS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset << " Bytes: " << mRequestedSize << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth << LL_ENDL; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); - mState = WAIT_HTTP_REQ; - - mFetcher->addToHTTPQueue(mID); - if (! mMetricsStartTime) - { - mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); - } - LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, - true, - LLImageBase::TYPE_AVATAR_BAKE == mType); +// LL_WARNS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset +// << " Bytes: " << mRequestedSize +// << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth +// << LL_ENDL; // Will call callbackHttpGet when curl request completes - std::vector headers; - headers.push_back("Accept: image/x-j2c"); - res = mFetcher->mCurlGetRequest->getByteRange(mUrl, headers, offset, mRequestedSize, - new HTTPGetResponder(mFetcher, mID, LLTimer::getTotalTime(), mRequestedSize, offset, true)); + // *FIXME: enable redirection follow + mHttpHandle = mFetcher->mHttpRequest->requestGetByteRange(mHttpPolicyClass, + mRequestedPriority, + mUrl, + mRequestedOffset, + mRequestedSize, + mFetcher->mHttpOptions, + mFetcher->mHttpHeaders, + this); } - if (!res) + if (LLCORE_HTTP_HANDLE_INVALID == mHttpHandle) { llwarns << "HTTP GET request failed for " << mID << llendl; resetFormattedData(); ++mHTTPFailCount; return true; // failed } + + mHttpActive = true; + mFetcher->addToHTTPQueue(mID); + recordTextureStart(true); + setPriority(LLWorkerThread::PRIORITY_HIGH); + mState = WAIT_HTTP_REQ; + // fall through } else //can not use http fetch. @@ -1219,27 +1148,28 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WAIT_HTTP_REQ) { + setPriority(0); if (mLoaded) { S32 cur_size = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; if (mRequestedSize < 0) { S32 max_attempts; - if (mGetStatus == HTTP_NOT_FOUND) + if (mGetStatus == LLCore::HttpStatus(HTTP_NOT_FOUND, LLCore::HE_REPLY_ERROR)) { mHTTPFailCount = max_attempts = 1; // Don't retry llwarns << "Texture missing from server (404): " << mUrl << llendl; //roll back to try UDP - if(mCanUseNET) + if (mCanUseNET) { - mState = INIT ; - mCanUseHTTP = false ; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); - return false ; + mState = INIT; + mCanUseHTTP = false; + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + return false; } } - else if (mGetStatus == HTTP_SERVICE_UNAVAILABLE) + else if (mGetStatus == LLCore::HttpStatus(HTTP_SERVICE_UNAVAILABLE, LLCore::HE_REPLY_ERROR)) { // *TODO: Should probably introduce a timer here to delay future HTTP requsts // for a short time (~1s) to ease server load? Ideally the server would queue @@ -1254,7 +1184,7 @@ bool LLTextureFetchWorker::doWork(S32 param) max_attempts = HTTP_MAX_RETRY_COUNT + 1; ++mHTTPFailCount; llinfos << "HTTP GET failed for: " << mUrl - << " Status: " << mGetStatus << " Reason: '" << mGetReason << "'" + << " Status: " << mGetStatus.toULong() << " Reason: '" << mGetReason << "'" << " Attempt:" << mHTTPFailCount+1 << "/" << max_attempts << llendl; } @@ -1264,12 +1194,14 @@ bool LLTextureFetchWorker::doWork(S32 param) { // Use available data mLoadedDiscard = mFormattedImage->getDiscardLevel(); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; return false; } else { resetFormattedData(); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; return true; // failed } @@ -1281,17 +1213,25 @@ bool LLTextureFetchWorker::doWork(S32 param) } } - llassert_always(mBufferSize == cur_size + mRequestedSize); - if(!mBufferSize)//no data received. + if (! mHttpBufferArray || ! mHttpBufferArray->size()) { - FREE_MEM(LLImageBase::getPrivatePool(), mBuffer); - mBuffer = NULL; + // no data received. + if (mHttpBufferArray) + { + mHttpBufferArray->release(); + mHttpBufferArray = NULL; + } - //abort. + // abort. + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; return true; } + const S32 append_size(mHttpBufferArray->size()); + const S32 total_size(cur_size + append_size); + llassert_always(append_size == mRequestedSize); + if (mFormattedImage.isNull()) { // For now, create formatted image based on extension @@ -1305,49 +1245,52 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mHaveAllData && mRequestedDiscard == 0) //the image file is fully loaded. { - mFileSize = mBufferSize; + mFileSize = total_size; } else //the file size is unknown. { - mFileSize = mBufferSize + 1 ; //flag the file is not fully loaded. + mFileSize = total_size + 1 ; //flag the file is not fully loaded. } - U8* buffer = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), mBufferSize); + U8 * buffer = (U8 *) ALLOCATE_MEM(LLImageBase::getPrivatePool(), total_size); if (cur_size > 0) { memcpy(buffer, mFormattedImage->getData(), cur_size); } - memcpy(buffer + cur_size, mBuffer, mRequestedSize); // append + mHttpBufferArray->seek(0); + mHttpBufferArray->read((char *) buffer + cur_size, append_size); + // NOTE: setData releases current data and owns new data (buffer) - mFormattedImage->setData(buffer, mBufferSize); - // delete temp data - FREE_MEM(LLImageBase::getPrivatePool(), mBuffer); // Note: not 'buffer' (assigned in setData()) - mBuffer = NULL; - mBufferSize = 0; + mFormattedImage->setData(buffer, total_size); + + // Done with buffer array + mHttpBufferArray->release(); + mHttpBufferArray = NULL; + mLoadedDiscard = mRequestedDiscard; + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; - if(mWriteToCacheState != NOT_WRITE) + if (mWriteToCacheState != NOT_WRITE) { mWriteToCacheState = SHOULD_WRITE ; } - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return false; } else { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; } } if (mState == DECODE_IMAGE) { + setPriority(0); static LLCachedControl textures_decode_disabled(gSavedSettings,"TextureDecodeDisabled"); - if(textures_decode_disabled) + if (textures_decode_disabled) { // for debug use, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } @@ -1355,7 +1298,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { // We aborted, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } @@ -1365,7 +1308,7 @@ bool LLTextureFetchWorker::doWork(S32 param) //abort, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } if (mLoadedDiscard < 0) @@ -1374,10 +1317,10 @@ bool LLTextureFetchWorker::doWork(S32 param) //abort, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + mRawImage = NULL; mAuxImage = NULL; llassert_always(mFormattedImage.notNull()); @@ -1394,6 +1337,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DECODE_IMAGE_UPDATE) { + setPriority(0); if (mDecoded) { if (mDecodedDiscard < 0) @@ -1406,13 +1350,14 @@ bool LLTextureFetchWorker::doWork(S32 param) llassert_always(mDecodeHandle == 0); mFormattedImage = NULL; ++mRetryAttempt; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); mState = INIT; return false; } else { // llwarns << "UNABLE TO LOAD TEXTURE: " << mID << " RETRIES: " << mRetryAttempt << llendl; + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; // failed } } @@ -1421,7 +1366,7 @@ bool LLTextureFetchWorker::doWork(S32 param) llassert_always(mRawImage.notNull()); LL_DEBUGS("Texture") << mID << ": Decoded. Discard: " << mDecodedDiscard << " Raw Image: " << llformat("%dx%d",mRawImage->getWidth(),mRawImage->getHeight()) << LL_ENDL; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = WRITE_TO_CACHE; } // fall through @@ -1434,10 +1379,12 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WRITE_TO_CACHE) { + setPriority(0); if (mWriteToCacheState != SHOULD_WRITE || mFormattedImage.isNull()) { // If we're in a local cache or we didn't actually receive any new data, // or we failed to load anything, skip + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; return false; } @@ -1457,6 +1404,7 @@ bool LLTextureFetchWorker::doWork(S32 param) setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it U32 cache_priority = mWorkPriority; mWritten = FALSE; + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = WAIT_ON_WRITE; CacheWriteResponder* responder = new CacheWriteResponder(mFetcher, mID); mCacheWriteHandle = mFetcher->mTextureCache->writeToCache(mID, cache_priority, @@ -1467,8 +1415,10 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WAIT_ON_WRITE) { + setPriority(0); if (writeToCacheComplete()) { + setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; // fall through } @@ -1487,16 +1437,15 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DONE) { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); if (mDecodedDiscard >= 0 && mDesiredDiscard < mDecodedDiscard) { // More data was requested, return to INIT mState = INIT; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return false; } else { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return true; } } @@ -1504,6 +1453,71 @@ bool LLTextureFetchWorker::doWork(S32 param) return false; } +// virtual +void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) +{ + static LLCachedControl log_to_viewer_log(gSavedSettings, "LogTextureDownloadsToViewerLog"); + static LLCachedControl log_to_sim(gSavedSettings, "LogTextureDownloadsToSimulator"); + static LLCachedControl log_texture_traffic(gSavedSettings, "LogTextureNetworkTraffic") ; + + mHttpActive = false; + + if (log_to_viewer_log || log_to_sim) + { + U64 timeNow = LLTimer::getTotalTime(); + mFetcher->mTextureInfo.setRequestStartTime(mID, mMetricsStartTime); + mFetcher->mTextureInfo.setRequestType(mID, LLTextureInfoDetails::REQUEST_TYPE_HTTP); + mFetcher->mTextureInfo.setRequestSize(mID, mRequestedSize); + mFetcher->mTextureInfo.setRequestOffset(mID, mRequestedOffset); + mFetcher->mTextureInfo.setRequestCompleteTimeAndLog(mID, timeNow); + } + + bool success = true; + bool partial = false; + LLCore::HttpStatus status(response->getStatus()); + + lldebugs << "HTTP COMPLETE: " << mID + << " status: " << status.toULong() << " '" << status.toString() << "'" + << llendl; + unsigned int offset(0), length(0); + response->getRange(&offset, &length); +// llwarns << "HTTP COMPLETE: " << mID << " handle: " << handle +// << " status: " << status.toULong() << " '" << status.toString() << "'" +// << " req offset: " << mRequestedOffset << " req length: " << mRequestedSize +// << " offset: " << offset << " length: " << length +// << llendl; + + if (! status) + { + success = false; + std::string reason(status.toString()); + setGetStatus(status, reason); + llwarns << "CURL GET FAILED, status:" << status.toULong() << " reason:" << reason << llendl; + } + else + { + static const LLCore::HttpStatus par_status(LLCore::HttpStatus(HTTP_PARTIAL_CONTENT, LLCore::HE_SUCCESS)); + + partial = (par_status == status); + } + + S32 data_size = callbackHttpGet(response, partial, success); + + if (log_texture_traffic && data_size > 0) + { + LLViewerTexture* tex = LLViewerTextureManager::findTexture(mID); + if (tex) + { + gTotalTextureBytesPerBoostLevel[tex->getBoostLevel()] += data_size ; + } + } + + mFetcher->removeFromHTTPQueue(mID); + + recordTextureDone(true); +} + + // Called from MAIN thread void LLTextureFetchWorker::endWork(S32 param, bool aborted) { @@ -1537,6 +1551,14 @@ void LLTextureFetchWorker::finishWork(S32 param, bool completed) bool LLTextureFetchWorker::deleteOK() { bool delete_ok = true; + + if (mHttpActive) + { + // HTTP library has a pointer to this worker + // and will dereference it to do notification. + delete_ok = false; + } + // Allow any pending reads or writes to complete if (mCacheReadHandle != LLTextureCache::nullHandle()) { @@ -1642,9 +1664,8 @@ bool LLTextureFetchWorker::processSimulatorPackets() ////////////////////////////////////////////////////////////////////////////// -S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, - const LLIOPipe::buffer_ptr_t& buffer, - bool partial, bool success) +S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, + bool partial, bool success) { S32 data_size = 0 ; @@ -1664,15 +1685,22 @@ S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, if (success) { // get length of stream: - data_size = buffer->countAfter(channels.in(), NULL); - + LLCore::BufferArray * body(response->getBody()); + data_size = body ? body->size() : 0; + LL_DEBUGS("Texture") << "HTTP RECEIVED: " << mID.asString() << " Bytes: " << data_size << LL_ENDL; if (data_size > 0) { // *TODO: set the formatted image data here directly to avoid the copy - mBuffer = (U8*)ALLOCATE_MEM(LLImageBase::getPrivatePool(), data_size); - buffer->readAfter(channels.in(), NULL, mBuffer, data_size); - mBufferSize += data_size; + // *FIXME: deal with actual offset and actual datasize, don't assume + // server gave exactly what was asked for. + + llassert_always(NULL == mHttpBufferArray); + + // Hold on to body for later copy + body->addRef(); + mHttpBufferArray = body; + if (data_size < mRequestedSize && mRequestedDiscard == 0) { mHaveAllData = TRUE; @@ -1684,7 +1712,6 @@ S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, mHaveAllData = TRUE; llassert_always(mDecodeHandle == 0); mFormattedImage = NULL; // discard any previous data we had - mBufferSize = data_size; } } else @@ -1700,7 +1727,7 @@ S32 LLTextureFetchWorker::callbackHttpGet(const LLChannelDescriptors& channels, mRequestedSize = -1; // error } mLoaded = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); return data_size ; } @@ -1729,7 +1756,7 @@ void LLTextureFetchWorker::callbackCacheRead(bool success, LLImageFormatted* ima } } mLoaded = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); } void LLTextureFetchWorker::callbackCacheWrite(bool success) @@ -1741,7 +1768,7 @@ void LLTextureFetchWorker::callbackCacheWrite(bool success) return; } mWritten = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); } ////////////////////////////////////////////////////////////////////////////// @@ -1779,7 +1806,7 @@ void LLTextureFetchWorker::callbackDecoded(bool success, LLImageRaw* raw, LLImag } mDecoded = TRUE; // llinfos << mID << " : DECODE COMPLETE " << llendl; - setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH); } ////////////////////////////////////////////////////////////////////////////// @@ -1806,6 +1833,34 @@ bool LLTextureFetchWorker::writeToCacheComplete() } +void LLTextureFetchWorker::recordTextureStart(bool is_http) +{ + if (! mMetricsStartTime) + { + mMetricsStartTime = LLViewerAssetStatsFF::get_timestamp(); + } + LLViewerAssetStatsFF::record_enqueue_thread1(LLViewerAssetType::AT_TEXTURE, + is_http, + LLImageBase::TYPE_AVATAR_BAKE == mType); +} + + +void LLTextureFetchWorker::recordTextureDone(bool is_http) +{ + if (mMetricsStartTime) + { + LLViewerAssetStatsFF::record_response_thread1(LLViewerAssetType::AT_TEXTURE, + is_http, + LLImageBase::TYPE_AVATAR_BAKE == mType, + LLViewerAssetStatsFF::get_timestamp() - mMetricsStartTime); + mMetricsStartTime = 0; + } + LLViewerAssetStatsFF::record_dequeue_thread1(LLViewerAssetType::AT_TEXTURE, + is_http, + LLImageBase::TYPE_AVATAR_BAKE == mType); +} + + ////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////// // public @@ -1823,17 +1878,26 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mTextureBandwidth(0), mHTTPTextureBits(0), mTotalHTTPRequests(0), - mCurlGetRequest(NULL), - mQAMode(qa_mode) + mQAMode(qa_mode), + mHttpRequest(NULL), + mHttpOptions(NULL), + mHttpHeaders(NULL) { mCurlPOSTRequestCount = 0; mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), gSavedSettings.getU32("TextureLoggingThreshold")); + + mHttpRequest = new LLCore::HttpRequest; + mHttpOptions = new LLCore::HttpOptions; + mHttpHeaders = new LLCore::HttpHeaders; + mHttpHeaders->mHeaders.push_back("Accept: image/x-j2c"); } LLTextureFetch::~LLTextureFetch() { - clearDeleteList() ; + cancelHttpRequests(); + + clearDeleteList(); while (! mCommands.empty()) { @@ -1841,7 +1905,22 @@ LLTextureFetch::~LLTextureFetch() mCommands.erase(mCommands.begin()); delete req; } - + + if (mHttpOptions) + { + mHttpOptions->release(); + mHttpOptions = NULL; + } + + if (mHttpHeaders) + { + mHttpHeaders->release(); + mHttpHeaders = NULL; + } + + delete mHttpRequest; + mHttpRequest = NULL; + // ~LLQueuedThread() called here } @@ -1984,6 +2063,28 @@ void LLTextureFetch::removeFromHTTPQueue(const LLUUID& id, S32 received_size) mHTTPTextureBits += received_size * 8; // Approximate - does not include header bits } +void LLTextureFetch::cancelHttpRequests() +{ + for (queue_t::iterator iter(mHTTPTextureQueue.begin()); + mHTTPTextureQueue.end() != iter; + ++iter) + { + LLTextureFetchWorker* worker = getWorker(*iter); + if (worker && worker->mHttpActive) + { + mHttpRequest->requestCancel(worker->mHttpHandle, NULL); + } + } + + // *FIXME: Do this better with less time wasting. + int tries(10); + while (! mHTTPTextureQueue.empty() && --tries) + { + mHttpRequest->update(100); + ms_sleep(100); + } +} + void LLTextureFetch::deleteRequest(const LLUUID& id, bool cancel) { lockQueue() ; @@ -2194,11 +2295,21 @@ void LLTextureFetch::commonUpdate() cmdDoWork(); // Update Curl on same thread as mCurlGetRequest was constructed - S32 processed = mCurlGetRequest->process(); + LLCore::HttpStatus status = mHttpRequest->update(200); + if (! status) + { + LL_INFOS_ONCE("Texture") << "Problem during HTTP servicing. Reason: " + << status.toString() + << LL_ENDL; + } + +#if 0 + // *FIXME: maybe implement this another way... if (processed > 0) { lldebugs << "processed: " << processed << " messages." << llendl; } +#endif } @@ -2256,22 +2367,22 @@ void LLTextureFetch::shutDownImageDecodeThread() // WORKER THREAD void LLTextureFetch::startThread() { - // Construct mCurlGetRequest from Worker Thread - mCurlGetRequest = new LLCurlRequest(); } // WORKER THREAD +// +// This detaches the texture fetch thread from the LLCore +// HTTP library but doesn't stop the thread running in that +// library... void LLTextureFetch::endThread() { - // Destroy mCurlGetRequest from Worker Thread - delete mCurlGetRequest; - mCurlGetRequest = NULL; + cancelHttpRequests(); } // WORKER THREAD void LLTextureFetch::threadedUpdate() { - llassert_always(mCurlGetRequest); + llassert_always(mHttpRequest); // Limit update frequency const F32 PROCESS_TIME = 0.05f; @@ -2579,7 +2690,7 @@ bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 llassert_always(totalbytes > 0); llassert_always(data_size == FIRST_PACKET_SIZE || data_size == worker->mFileSize); res = worker->insertPacket(0, data, data_size); - worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); + worker->setPriority(LLWorkerThread::PRIORITY_HIGH); worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; worker->unlockWorkMutex(); return res; @@ -2623,7 +2734,7 @@ bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U1 if ((worker->mState == LLTextureFetchWorker::LOAD_FROM_SIMULATOR) || (worker->mState == LLTextureFetchWorker::LOAD_FROM_NETWORK)) { - worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); + worker->setPriority(LLWorkerThread::PRIORITY_HIGH); worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; } else @@ -2730,6 +2841,14 @@ void LLTextureFetch::dump() << " STATE: " << worker->sStateDescs[worker->mState] << llendl; } + + llinfos << "LLTextureFetch ACTIVE_HTTP:" << llendl; + for (queue_t::const_iterator iter(mHTTPTextureQueue.begin()); + mHTTPTextureQueue.end() != iter; + ++iter) + { + llinfos << " ID: " << (*iter) << llendl; + } } ////////////////////////////////////////////////////////////////////////////// @@ -2942,6 +3061,8 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) if (! mCapsURL.empty()) { LLCurlRequest::headers_t headers; +#if 0 + // *FIXME: Going to need a post op after all... fetcher->getCurlRequest().post(mCapsURL, headers, merged_llsd, @@ -2950,6 +3071,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) report_sequence, LLTextureFetch::svMetricsDataBreak, reporting_started)); +#endif } else { diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 35df7d816f..402b198246 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2000&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2012, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -31,9 +31,11 @@ #include "llimage.h" #include "lluuid.h" #include "llworkerthread.h" -#include "llcurl.h" #include "lltextureinfo.h" #include "llapr.h" +#include "httprequest.h" +#include "httpoptions.h" +#include "httpheaders.h" class LLViewerTexture; class LLTextureFetchWorker; @@ -98,7 +100,7 @@ public: LLViewerAssetStats * main_stats); void commandDataBreak(); - LLCurlRequest & getCurlRequest() { return *mCurlGetRequest; } + LLCore::HttpRequest & getHttpRequest() { return *mHttpRequest; } bool isQAMode() const { return mQAMode; } @@ -112,7 +114,8 @@ protected: void addToHTTPQueue(const LLUUID& id); void removeFromHTTPQueue(const LLUUID& id, S32 received_size = 0); void removeRequest(LLTextureFetchWorker* worker, bool cancel); - + void cancelHttpRequests(); + // Overrides from the LLThread tree bool runCondition(); @@ -166,7 +169,6 @@ private: LLTextureCache* mTextureCache; LLImageDecodeThread* mImageDecodeThread; - LLCurlRequest* mCurlGetRequest; // Map of all requests by UUID typedef std::map map_t; @@ -203,6 +205,13 @@ private: // use the LLCurl module's request counter as it isn't thread compatible. // *NOTE: Don't mix Atomic and static, apr_initialize must be called first. LLAtomic32 mCurlPOSTRequestCount; + + // Interfaces and objects into the core http library used + // to make our HTTP requests. These replace the various + // LLCurl interfaces used in the past. + LLCore::HttpRequest * mHttpRequest; + LLCore::HttpOptions * mHttpOptions; + LLCore::HttpHeaders * mHttpHeaders; public: // A probabilistically-correct indicator that the current -- cgit v1.3 From b8edacd0bb4feacc3ac1d61421e600c75ab87f7c Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 1 Jun 2012 14:07:34 -0400 Subject: Major steps towards implementing the policy component. Identified and reacted to the priority inversion problem we have in texturefetch. Includes the introduction of a priority_queue for the requests that are ready. Start some parameterization in anticipation of having policy_class everywhere. Removed _assert.h which isn't really needed in indra codebase. Implemented async setPriority request (which I hope I can get rid of eventually along with all priorities in this library). Converted to using unsigned int for priority rather than float. Implemented POST and did groundwork for PUT. --- indra/llcorehttp/CMakeLists.txt | 27 ++-- indra/llcorehttp/_assert.h | 39 ----- indra/llcorehttp/_httplibcurl.cpp | 52 ++++-- indra/llcorehttp/_httplibcurl.h | 27 +++- indra/llcorehttp/_httpopcancel.h | 2 +- indra/llcorehttp/_httpoperation.cpp | 8 +- indra/llcorehttp/_httpoperation.h | 42 ++++- indra/llcorehttp/_httpoprequest.cpp | 113 ++++++++++++- indra/llcorehttp/_httpoprequest.h | 11 +- indra/llcorehttp/_httpopsetpriority.cpp | 77 +++++++++ indra/llcorehttp/_httpopsetpriority.h | 70 +++++++++ indra/llcorehttp/_httppolicy.cpp | 97 ++++++++++-- indra/llcorehttp/_httppolicy.h | 28 +++- indra/llcorehttp/_httpreadyqueue.h | 85 ++++++++++ indra/llcorehttp/_httprequestqueue.cpp | 2 +- indra/llcorehttp/_httpservice.cpp | 63 +++++--- indra/llcorehttp/_httpservice.h | 26 ++- indra/llcorehttp/_refcounted.h | 14 +- indra/llcorehttp/httpcommon.cpp | 15 +- indra/llcorehttp/httpcommon.h | 8 +- indra/llcorehttp/httprequest.cpp | 52 +++++- indra/llcorehttp/httprequest.h | 49 +++++- indra/llcorehttp/tests/test_httprequest.hpp | 2 +- indra/newview/lltexturefetch.cpp | 235 ++++++++++++---------------- indra/newview/lltexturefetch.h | 15 -- 25 files changed, 869 insertions(+), 290 deletions(-) delete mode 100644 indra/llcorehttp/_assert.h create mode 100644 indra/llcorehttp/_httpopsetpriority.cpp create mode 100644 indra/llcorehttp/_httpopsetpriority.h create mode 100644 indra/llcorehttp/_httpreadyqueue.h (limited to 'indra/llcorehttp/httpcommon.cpp') diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt index ae92fb96fd..85df5364db 100644 --- a/indra/llcorehttp/CMakeLists.txt +++ b/indra/llcorehttp/CMakeLists.txt @@ -23,17 +23,18 @@ include_directories( set(llcorehttp_SOURCE_FILES bufferarray.cpp httpcommon.cpp + httpheaders.cpp + httpoptions.cpp httprequest.cpp httpresponse.cpp - httpoptions.cpp - httpheaders.cpp - _httprequestqueue.cpp + _httplibcurl.cpp + _httpopcancel.cpp _httpoperation.cpp _httpoprequest.cpp - _httpopcancel.cpp - _httpreplyqueue.cpp + _httpopsetpriority.cpp _httppolicy.cpp - _httplibcurl.cpp + _httpreplyqueue.cpp + _httprequestqueue.cpp _httpservice.cpp _refcounted.cpp ) @@ -48,17 +49,19 @@ set(llcorehttp_HEADER_FILES httpoptions.h httprequest.h httpresponse.h + _assert.h + _httplibcurl.h + _httpopcancel.h _httpoperation.h _httpoprequest.h - _httpopcancel.h - _httprequestqueue.h + _httpopsetpriority.h + _httppolicy.h + _httpreadyqueue.h _httpreplyqueue.h + _httprequestqueue.h _httpservice.h - _httppolicy.h - _httplibcurl.h - _assert.h - _refcounted.h _mutex.h + _refcounted.h _thread.h ) diff --git a/indra/llcorehttp/_assert.h b/indra/llcorehttp/_assert.h deleted file mode 100644 index 054f23ef32..0000000000 --- a/indra/llcorehttp/_assert.h +++ /dev/null @@ -1,39 +0,0 @@ -/** - * @file _assert - * @brief assert abstraction - * - * $LicenseInfo:firstyear=2012&license=viewerlgpl$ - * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License only. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA - * $/LicenseInfo$ - */ - -#ifndef LLCOREINT__ASSERT_H_ -#define LLCOREINT__ASSERT_H_ - -#ifdef DEBUG_ASSERT -#include -#define LLINT_ASSERT(x) assert(x) -#else -#define LLINT_ASSERT(x) -#endif - -#endif // LLCOREINT__ASSERT_H_ - - diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 1b951818e4..704f9baac9 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -28,7 +28,6 @@ #include "httpheaders.h" #include "_httpoprequest.h" -#include "_httpservice.h" namespace LLCore @@ -38,6 +37,12 @@ namespace LLCore HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service) { + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + { + mMultiHandles[policy_class] = 0; + } + + // Create multi handle for default class mMultiHandles[0] = curl_multi_init(); } @@ -51,15 +56,18 @@ HttpLibcurl::~HttpLibcurl() (*item)->cancel(); (*item)->release(); + mActiveOps.erase(item); } - if (mMultiHandles[0]) + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - // *FIXME: Do some multi cleanup here first - + if (mMultiHandles[policy_class]) + { + // *FIXME: Do some multi cleanup here first - curl_multi_cleanup(mMultiHandles[0]); - mMultiHandles[0] = NULL; + curl_multi_cleanup(mMultiHandles[policy_class]); + mMultiHandles[policy_class] = 0; + } } mService = NULL; @@ -74,31 +82,34 @@ void HttpLibcurl::term() {} -void HttpLibcurl::processTransport() +HttpService::ELoopSpeed HttpLibcurl::processTransport() { - if (mMultiHandles[0]) + // Give libcurl some cycles to do I/O & callbacks + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - // Give libcurl some cycles to do I/O & callbacks + if (! mMultiHandles[policy_class]) + continue; + int running(0); CURLMcode status(CURLM_CALL_MULTI_PERFORM); do { running = 0; - status = curl_multi_perform(mMultiHandles[0], &running); + status = curl_multi_perform(mMultiHandles[policy_class], &running); } while (0 != running && CURLM_CALL_MULTI_PERFORM == status); // Run completion on anything done CURLMsg * msg(NULL); int msgs_in_queue(0); - while ((msg = curl_multi_info_read(mMultiHandles[0], &msgs_in_queue))) + while ((msg = curl_multi_info_read(mMultiHandles[policy_class], &msgs_in_queue))) { if (CURLMSG_DONE == msg->msg) { CURL * handle(msg->easy_handle); CURLcode result(msg->data.result); - completeRequest(mMultiHandles[0], handle, result); + completeRequest(mMultiHandles[policy_class], handle, result); handle = NULL; // No longer valid on return } else if (CURLMSG_NONE == msg->msg) @@ -114,13 +125,18 @@ void HttpLibcurl::processTransport() msgs_in_queue = 0; } } + + return mActiveOps.empty() ? HttpService::REQUEST_SLEEP : HttpService::NORMAL; } void HttpLibcurl::addOp(HttpOpRequest * op) { + llassert_always(op->mReqPolicy < HttpRequest::POLICY_CLASS_LIMIT); + llassert_always(mMultiHandles[op->mReqPolicy] != NULL); + // Create standard handle - if (! op->prepareForGet(mService)) + if (! op->prepareRequest(mService)) { // Couldn't issue request, fail with notification // *FIXME: Need failure path @@ -128,7 +144,7 @@ void HttpLibcurl::addOp(HttpOpRequest * op) } // Make the request live - curl_multi_add_handle(mMultiHandles[0], op->mCurlHandle); + curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); op->mCurlActive = true; // On success, make operation active @@ -190,12 +206,18 @@ void HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode } -int HttpLibcurl::activeCount() const +int HttpLibcurl::getActiveCount() const { return mActiveOps.size(); } +int HttpLibcurl::getActiveCountInClass(int /* policy_class */) const +{ + return getActiveCount(); +} + + // --------------------------------------- // Free functions // --------------------------------------- diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 807196628d..ec325c1946 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -34,18 +34,25 @@ #include +#include "httprequest.h" +#include "_httpservice.h" + namespace LLCore { -class HttpService; class HttpPolicy; class HttpOpRequest; class HttpHeaders; /// Implements libcurl-based transport for an HttpService instance. +/// +/// Threading: Single-threaded. Other than for construction/destruction, +/// all methods are expected to be invoked in a single thread, typically +/// a worker thread of some sort. + class HttpLibcurl { public: @@ -60,12 +67,22 @@ public: static void init(); static void term(); - void processTransport(); - void addOp(HttpOpRequest * op); + /// Give cycles to libcurl to run active requests. Completed + /// operations (successful or failed) will be retried or handed + /// over to the reply queue as final responses. + HttpService::ELoopSpeed processTransport(); - int activeCount() const; + /// Add request to the active list. Caller is expected to have + /// provided us with a reference count to hold the request. (No + /// additional references will be added.) + void addOp(HttpOpRequest * op); + int getActiveCount() const; + int getActiveCountInClass(int policy_class) const; + protected: + /// Invoked when libcurl has indicated a request has been processed + /// to completion and we need to move the request to a new state. void completeRequest(CURLM * multi_handle, CURL * handle, CURLcode status); protected: @@ -74,7 +91,7 @@ protected: protected: HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; - CURLM * mMultiHandles[1]; + CURLM * mMultiHandles[HttpRequest::POLICY_CLASS_LIMIT]; }; // end class HttpLibcurl diff --git a/indra/llcorehttp/_httpopcancel.h b/indra/llcorehttp/_httpopcancel.h index 38ccc585ed..fab6f1f362 100644 --- a/indra/llcorehttp/_httpopcancel.h +++ b/indra/llcorehttp/_httpopcancel.h @@ -45,7 +45,7 @@ namespace LLCore /// HttpOpCancel requests that a previously issued request /// be canceled, if possible. Requests that have been made /// active and are available for sending on the wire cannot -/// be canceled. +/// be canceled. class HttpOpCancel : public HttpOperation { diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 17c65b0379..d966efd12b 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -50,7 +50,7 @@ HttpOperation::HttpOperation() mLibraryHandler(NULL), mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), - mReqPriority(0.0f) + mReqPriority(0U) { } @@ -95,7 +95,7 @@ void HttpOperation::stageFromRequest(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LLINT_ASSERT(false); + llassert_always(false); } @@ -105,7 +105,7 @@ void HttpOperation::stageFromReady(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LLINT_ASSERT(false); + llassert_always(false); } @@ -115,7 +115,7 @@ void HttpOperation::stageFromActive(HttpService *) // Default implementation should never be called. This // indicates an operation making a transition that isn't // defined. - LLINT_ASSERT(false); + llassert_always(false); } diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index 5d06a28586..6c0c3183b7 100644 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -44,6 +44,32 @@ class HttpRequest; /// HttpOperation is the base class for all request/reply /// pairs. /// +/// Operations are expected to be of two types: immediate +/// and queued. Immediate requests go to the singleton +/// request queue and when picked up by the worker thread +/// are executed immediately and there results placed on +/// the supplied reply queue. Queued requests (namely for +/// HTTP operations), go to the request queue, are picked +/// up and moved to a ready queue where they're ordered by +/// priority and managed by the policy component, are +/// then activated issuing HTTP requests and moved to an +/// active list managed by the transport (libcurl) component +/// and eventually finalized when a response is available +/// and status and data return via reply queue. +/// +/// To manage these transitions, derived classes implement +/// three methods: stageFromRequest, stageFromReady and +/// stageFromActive. Immediate requests will only override +/// stageFromRequest which will perform the operation and +/// return the result by invoking addAsReply() to put the +/// request on a reply queue. Queued requests will involve +/// all three stage methods. +/// +/// Threading: not thread-safe. Base and derived classes +/// provide no locking. Instances move across threads +/// via queue-like interfaces that are thread compatible +/// and those interfaces establish the access rules. + class HttpOperation : public LLCoreInt::RefCounted { public: @@ -82,7 +108,7 @@ protected: public: unsigned int mReqPolicy; - float mReqPriority; + unsigned int mReqPriority; }; // end class HttpOperation @@ -133,6 +159,20 @@ public: }; // end class HttpOpNull + +/// HttpOpCompare isn't an operation but a uniform comparison +/// functor for STL containers that order by priority. Mainly +/// used for the ready queue container but defined here. +class HttpOpCompare +{ +public: + bool operator()(const HttpOperation * lhs, const HttpOperation * rhs) + { + return lhs->mReqPriority > rhs->mReqPriority; + } +}; // end class HttpOpCompare + + } // end namespace LLCore #endif // _LLCORE_HTTP_OPERATION_H_ diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 521bd5b879..54b9990057 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -93,6 +93,7 @@ HttpOpRequest::HttpOpRequest() mCurlHandle(NULL), mCurlService(NULL), mCurlHeaders(NULL), + mCurlBodyPos(0), mReplyBody(NULL), mReplyOffset(0), mReplyLength(0), @@ -267,12 +268,46 @@ HttpStatus HttpOpRequest::setupGetByteRange(unsigned int policy_id, } -HttpStatus HttpOpRequest::prepareForGet(HttpService * service) +HttpStatus HttpOpRequest::setupPost(unsigned int policy_id, + float priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers) +{ + HttpStatus status; + + mProcFlags = 0; + mReqPolicy = policy_id; + mReqPriority = priority; + mReqMethod = HOR_POST; + mReqURL = url; + if (body) + { + body->addRef(); + mReqBody = body; + } + if (headers && ! mReqHeaders) + { + headers->addRef(); + mReqHeaders = headers; + } + if (options && ! mReqOptions) + { + mReqOptions = new HttpOptions(*options); + } + + return status; +} + + +HttpStatus HttpOpRequest::prepareRequest(HttpService * service) { // *FIXME: better error handling later HttpStatus status; mCurlHandle = curl_easy_init(); + // curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30); curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, 30); curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1); @@ -280,20 +315,68 @@ HttpStatus HttpOpRequest::prepareForGet(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_URL, mReqURL.c_str()); curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this); curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); + // *FIXME: Need to deal with proxy setup... // curl_easy_setopt(handle, CURLOPT_PROXY, ""); + // *FIXME: Revisit this old DNS timeout setting - may no longer be valid curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); + curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle); + curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); + curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, mCurlHandle); + switch (mReqMethod) + { + case HOR_GET: + curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); + break; + + case HOR_POST: + { + curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); + long data_size(0); + if (mReqBody) + { + mReqBody->seek(0); + data_size = mReqBody->size(); + } + curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, static_cast(NULL)); + curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDSIZE, data_size); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); + } + break; + + case HOR_PUT: + { + curl_easy_setopt(mCurlHandle, CURLOPT_UPLOAD, 1); + long data_size(0); + if (mReqBody) + { + mReqBody->seek(0); + data_size = mReqBody->size(); + } + curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); + } + break; + + default: + // *FIXME: fail out here + break; + } + if (mReqHeaders) { mCurlHeaders = append_headers_to_slist(mReqHeaders, mCurlHeaders); } mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); - if (mReqOffset || mReqLength) + if ((mReqOffset || mReqLength) && HOR_GET == mReqMethod) { static const char * fmt1("Range: bytes=%d-%d"); static const char * fmt2("Range: bytes=%d-"); @@ -347,6 +430,32 @@ size_t HttpOpRequest::writeCallback(void * data, size_t size, size_t nmemb, void } +size_t HttpOpRequest::readCallback(void * data, size_t size, size_t nmemb, void * userdata) +{ + CURL * handle(static_cast(userdata)); + HttpOpRequest * op(NULL); + curl_easy_getinfo(handle, CURLINFO_PRIVATE, &op); + // *FIXME: check the pointer + + if (! op->mReqBody) + { + return 0; + } + const size_t req_size(size * nmemb); + const size_t body_size(op->mReqBody->size()); + if (body_size <= op->mCurlBodyPos) + { + // *FIXME: should probably log this event - unexplained + return 0; + } + + const size_t do_size((std::min)(req_size, body_size - op->mCurlBodyPos)); + op->mReqBody->read(static_cast(data), do_size); + op->mCurlBodyPos += do_size; + return do_size; +} + + size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, void * userdata) { static const char status_line[] = "HTTP/"; diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 601937a943..7efed0b1d9 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -84,12 +84,20 @@ public: HttpOptions * options, HttpHeaders * headers); - HttpStatus prepareForGet(HttpService * service); + HttpStatus setupPost(unsigned int policy_id, + float priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers); + + HttpStatus prepareRequest(HttpService * service); virtual HttpStatus cancel(); protected: static size_t writeCallback(void * data, size_t size, size_t nmemb, void * userdata); + static size_t readCallback(void * data, size_t size, size_t nmemb, void * userdata); static size_t headerCallback(void * data, size_t size, size_t nmemb, void * userdata); protected: @@ -112,6 +120,7 @@ public: CURL * mCurlHandle; HttpService * mCurlService; curl_slist * mCurlHeaders; + size_t mCurlBodyPos; // Result data HttpStatus mStatus; diff --git a/indra/llcorehttp/_httpopsetpriority.cpp b/indra/llcorehttp/_httpopsetpriority.cpp new file mode 100644 index 0000000000..dc609421ed --- /dev/null +++ b/indra/llcorehttp/_httpopsetpriority.cpp @@ -0,0 +1,77 @@ +/** + * @file _httpopsetpriority.cpp + * @brief Definitions for internal classes based on HttpOpSetPriority + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#include "_httpopsetpriority.h" + +#include "httpresponse.h" +#include "httphandler.h" +#include "_httpservice.h" + + +namespace LLCore +{ + + +HttpOpSetPriority::HttpOpSetPriority(HttpHandle handle, unsigned int priority) + : HttpOperation(), + mHandle(handle), + mPriority(priority) +{} + + +HttpOpSetPriority::~HttpOpSetPriority() +{} + + +void HttpOpSetPriority::stageFromRequest(HttpService * service) +{ + // Do operations + if (! service->changePriority(mHandle, mPriority)) + { + // Request not found, fail the final status + mStatus = HttpStatus(HttpStatus::LLCORE, HE_HANDLE_NOT_FOUND); + } + + // Move directly to response queue + addAsReply(); +} + + +void HttpOpSetPriority::visitNotifier(HttpRequest * request) +{ + if (mLibraryHandler) + { + HttpResponse * response = new HttpResponse(); + + response->setStatus(mStatus); + mLibraryHandler->onCompleted(static_cast(this), response); + + response->release(); + } +} + + +} // end namespace LLCore diff --git a/indra/llcorehttp/_httpopsetpriority.h b/indra/llcorehttp/_httpopsetpriority.h new file mode 100644 index 0000000000..e5d8e5fc1f --- /dev/null +++ b/indra/llcorehttp/_httpopsetpriority.h @@ -0,0 +1,70 @@ +/** + * @file _httpsetpriority.h + * @brief Internal declarations for HttpSetPriority + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_SETPRIORITY_H_ +#define _LLCORE_HTTP_SETPRIORITY_H_ + + +#include "httpcommon.h" + +#include "_httpoperation.h" +#include "_refcounted.h" + + +namespace LLCore +{ + + +/// HttpOpSetPriority is an immediate request that +/// searches the various queues looking for a given +/// request handle and changing it's priority if +/// found. + +class HttpOpSetPriority : public HttpOperation +{ +public: + HttpOpSetPriority(HttpHandle handle, unsigned int priority); + virtual ~HttpOpSetPriority(); + +private: + HttpOpSetPriority(const HttpOpSetPriority &); // Not defined + void operator=(const HttpOpSetPriority &); // Not defined + +public: + virtual void stageFromRequest(HttpService *); + + virtual void visitNotifier(HttpRequest * request); + +protected: + HttpStatus mStatus; + HttpHandle mHandle; + unsigned int mPriority; +}; // end class HttpOpSetPriority + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_SETPRIORITY_H_ + diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index d965a6cf3a..873b519c51 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -28,48 +28,113 @@ #include "_httpoprequest.h" #include "_httpservice.h" +#include "_httplibcurl.h" namespace LLCore { - HttpPolicy::HttpPolicy(HttpService * service) : mService(service) -{} +{ + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + { + mReadyInClass[policy_class] = 0; + } +} HttpPolicy::~HttpPolicy() { - for (ready_queue_t::reverse_iterator i(mReadyQueue.rbegin()); - mReadyQueue.rend() != i;) + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - ready_queue_t::reverse_iterator cur(i++); - - (*cur)->cancel(); - (*cur)->release(); + HttpReadyQueue & readyq(mReadyQueue[policy_class]); + + while (! readyq.empty()) + { + HttpOpRequest * op(readyq.top()); + + op->cancel(); + op->release(); + mReadyInClass[policy_class]--; + readyq.pop(); + } } - mService = NULL; } void HttpPolicy::addOp(HttpOpRequest * op) { - mReadyQueue.push_back(op); + const int policy_class(op->mReqPolicy); + + mReadyQueue[policy_class].push(op); + ++mReadyInClass[policy_class]; } -void HttpPolicy::processReadyQueue() +HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { - while (! mReadyQueue.empty()) + HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); + HttpLibcurl * pTransport(mService->getTransport()); + + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { - HttpOpRequest * op(mReadyQueue.front()); - mReadyQueue.erase(mReadyQueue.begin()); + HttpReadyQueue & readyq(mReadyQueue[policy_class]); + int active(pTransport->getActiveCountInClass(policy_class)); + int needed(8 - active); + + if (needed > 0 && mReadyInClass[policy_class] > 0) + { + // Scan ready queue for requests that match policy + + while (! readyq.empty() && needed > 0 && mReadyInClass[policy_class] > 0) + { + HttpOpRequest * op(readyq.top()); + readyq.pop(); + + op->stageFromReady(mService); + op->release(); + + --mReadyInClass[policy_class]; + --needed; + } + } + + if (! readyq.empty()) + { + // If anything is ready, continue looping... + result = (std::min)(result, HttpService::NORMAL); + } + } + + return result; +} - op->stageFromReady(mService); - op->release(); + +bool HttpPolicy::changePriority(HttpHandle handle, unsigned int priority) +{ + for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + { + HttpReadyQueue::container_type & c(mReadyQueue[policy_class].get_container()); + + // Scan ready queue for requests that match policy + for (HttpReadyQueue::container_type::iterator iter(c.begin()); c.end() != iter;) + { + HttpReadyQueue::container_type::iterator cur(iter++); + + if (static_cast(*cur) == handle) + { + HttpOpRequest * op(*cur); + c.erase(cur); // All iterators are now invalidated + op->mReqPriority = priority; + mReadyQueue[policy_class].push(op); // Re-insert using adapter class + return true; + } + } } + + return false; } diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 192bc73b31..c5e82d0a65 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -28,18 +28,23 @@ #define _LLCORE_HTTP_POLICY_H_ -#include +#include "httprequest.h" +#include "_httpservice.h" +#include "_httpreadyqueue.h" namespace LLCore { - -class HttpService; +class HttpReadyQueue; class HttpOpRequest; /// Implements class-based queuing policies for an HttpService instance. +/// +/// Threading: Single-threaded. Other than for construction/destruction, +/// all methods are expected to be invoked in a single thread, typically +/// a worker thread of some sort. class HttpPolicy { public: @@ -51,16 +56,23 @@ private: void operator=(const HttpPolicy &); // Not defined public: - void processReadyQueue(); + /// Give the policy layer some cycles to scan the ready + /// queue promoting higher-priority requests to active + /// as permited. + HttpService::ELoopSpeed processReadyQueue(); + /// Add request to a ready queue. Caller is expected to have + /// provided us with a reference count to hold the request. (No + /// additional references will be added.) void addOp(HttpOpRequest *); + + // Shadows HttpService's method + bool changePriority(HttpHandle handle, unsigned int priority); protected: - typedef std::vector ready_queue_t; - -protected: + int mReadyInClass[HttpRequest::POLICY_CLASS_LIMIT]; + HttpReadyQueue mReadyQueue[HttpRequest::POLICY_CLASS_LIMIT]; HttpService * mService; // Naked pointer, not refcounted, not owner - ready_queue_t mReadyQueue; }; // end class HttpPolicy diff --git a/indra/llcorehttp/_httpreadyqueue.h b/indra/llcorehttp/_httpreadyqueue.h new file mode 100644 index 0000000000..283e868b4c --- /dev/null +++ b/indra/llcorehttp/_httpreadyqueue.h @@ -0,0 +1,85 @@ +/** + * @file _httpreadyqueue.h + * @brief Internal declaration for the operation ready queue + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_READY_QUEUE_H_ +#define _LLCORE_HTTP_READY_QUEUE_H_ + + +#include + +#include "_httpoperation.h" + + +namespace LLCore +{ + +class HttpOpRequest; + +/// HttpReadyQueue provides a simple priority queue for HttpOpRequest objects. +/// +/// This uses the priority_queue adaptor class to provide the queue +/// as well as the ordering scheme while allowing us access to the +/// raw container if we follow a few simple rules. One of the more +/// important of those rules is that any iterator becomes invalid +/// on element erasure. So pay attention. +/// +/// Threading: not thread-safe. Expected to be used entirely by +/// a single thread, typically a worker thread of some sort. + +class HttpReadyQueue : public std::priority_queue, + LLCore::HttpOpCompare> +{ +public: + HttpReadyQueue() + : priority_queue() + {} + + ~HttpReadyQueue() + {} + +protected: + HttpReadyQueue(const HttpReadyQueue &); // Not defined + void operator=(const HttpReadyQueue &); // Not defined + +public: + const container_type & get_container() const + { + return c; + } + + container_type & get_container() + { + return c; + } + +}; // end class HttpReadyQueue + + +} // end namespace LLCore + + +#endif // _LLCORE_HTTP_READY_QUEUE_H_ diff --git a/indra/llcorehttp/_httprequestqueue.cpp b/indra/llcorehttp/_httprequestqueue.cpp index c36814aee3..92bb5ec5c1 100644 --- a/indra/llcorehttp/_httprequestqueue.cpp +++ b/indra/llcorehttp/_httprequestqueue.cpp @@ -57,7 +57,7 @@ HttpRequestQueue::~HttpRequestQueue() void HttpRequestQueue::init() { - LLINT_ASSERT(! sInstance); + llassert_always(! sInstance); sInstance = new HttpRequestQueue(); } diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 6ebc0ec6cb..48884ca060 100644 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -35,6 +35,8 @@ #include "_httplibcurl.h" #include "_thread.h" +#include "lltimer.h" + namespace LLCore { @@ -89,8 +91,8 @@ HttpService::~HttpService() void HttpService::init(HttpRequestQueue * queue) { - LLINT_ASSERT(! sInstance); - LLINT_ASSERT(NOT_INITIALIZED == sState); + llassert_always(! sInstance); + llassert_always(NOT_INITIALIZED == sState); sInstance = new HttpService(); queue->addRef(); @@ -103,7 +105,7 @@ void HttpService::init(HttpRequestQueue * queue) void HttpService::term() { - LLINT_ASSERT(RUNNING != sState); + llassert_always(RUNNING != sState); if (sInstance) { delete sInstance; @@ -132,8 +134,8 @@ bool HttpService::isStopped() void HttpService::startThread() { - LLINT_ASSERT(! mThread || STOPPED == sState); - LLINT_ASSERT(INITIALIZED == sState || STOPPED == sState); + llassert_always(! mThread || STOPPED == sState); + llassert_always(INITIALIZED == sState || STOPPED == sState); if (mThread) { @@ -150,6 +152,20 @@ void HttpService::stopRequested() mExitRequested = true; } +bool HttpService::changePriority(HttpHandle handle, unsigned int priority) +{ + bool found(false); + + // Skip the request queue as we currently don't leave earlier + // requests sitting there. Start with the ready queue... + found = mPolicy->changePriority(handle, priority); + + // If not there, we could try the transport/active queue but priority + // doesn't really have much effect there so we don't waste cycles. + + return found; +} + void HttpService::shutdown() { @@ -157,38 +173,46 @@ void HttpService::shutdown() } +// Working thread loop-forever method. Gives time to +// each of the request queue, policy layer and transport +// layer pieces and then either sleeps for a small time +// or waits for a request to come in. Repeats until +// requested to stop. void HttpService::threadRun(LLCoreInt::HttpThread * thread) { boost::this_thread::disable_interruption di; - + ELoopSpeed loop(REQUEST_SLEEP); + while (! mExitRequested) { - processRequestQueue(); + loop = processRequestQueue(loop); // Process ready queue issuing new requests as needed - mPolicy->processReadyQueue(); + ELoopSpeed new_loop = mPolicy->processReadyQueue(); + loop = (std::min)(loop, new_loop); // Give libcurl some cycles - mTransport->processTransport(); + new_loop = mTransport->processTransport(); + loop = (std::min)(loop, new_loop); // Determine whether to spin, sleep briefly or sleep for next request - // *FIXME: For now, do this -#if defined(WIN32) - Sleep(50); -#else - usleep(5000); -#endif + if (REQUEST_SLEEP != loop) + { + ms_sleep(50); + } } + shutdown(); sState = STOPPED; } -void HttpService::processRequestQueue() +HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) { HttpRequestQueue::OpContainer ops; - - mRequestQueue->fetchAll(false, ops); + const bool wait_for_req(REQUEST_SLEEP == loop); + + mRequestQueue->fetchAll(wait_for_req, ops); while (! ops.empty()) { HttpOperation * op(ops.front()); @@ -203,6 +227,9 @@ void HttpService::processRequestQueue() // Done with operation op->release(); } + + // Queue emptied, allow polling loop to sleep + return REQUEST_SLEEP; } diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index ba76e1eeca..3e5a5457d7 100644 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -28,6 +28,9 @@ #define _LLCORE_HTTP_SERVICE_H_ +#include "httpcommon.h" + + namespace LLCoreInt { @@ -86,7 +89,17 @@ public: RUNNING, ///< thread created and running STOPPED ///< thread has committed to exiting }; - + + // Ordered enumeration of idling strategies available to + // threadRun's loop. Ordered so that std::min on values + // produces the most conservative result of multiple + // requests. + enum ELoopSpeed + { + NORMAL, ///< continuous polling of request, ready, active queues + REQUEST_SLEEP ///< can sleep indefinitely waiting for request queue write + }; + static void init(HttpRequestQueue *); static void term(); @@ -124,6 +137,15 @@ public: /// Threading: callable by worker thread. void shutdown(); + + /// Try to find the given request handle on any of the request + /// queues and reset the priority (and queue position) of the + /// request if found. + /// + /// @return True if the request was found somewhere. + /// + /// Threading: callable by worker thread. + bool changePriority(HttpHandle handle, unsigned int priority); HttpPolicy * getPolicy() { @@ -138,7 +160,7 @@ public: protected: void threadRun(LLCoreInt::HttpThread * thread); - void processRequestQueue(); + ELoopSpeed processRequestQueue(ELoopSpeed loop); protected: static HttpService * sInstance; diff --git a/indra/llcorehttp/_refcounted.h b/indra/llcorehttp/_refcounted.h index 4a6ce8420a..72cef6b342 100644 --- a/indra/llcorehttp/_refcounted.h +++ b/indra/llcorehttp/_refcounted.h @@ -30,7 +30,7 @@ #include -#include "_assert.h" +#include "linden_common.h" namespace LLCoreInt @@ -71,7 +71,7 @@ private: inline void RefCounted::addRef() const { boost::mutex::scoped_lock lock(mRefLock); - LLINT_ASSERT(mRefCount >= 0); + llassert_always(mRefCount >= 0); ++mRefCount; } @@ -82,8 +82,8 @@ inline void RefCounted::release() const { // CRITICAL SECTION boost::mutex::scoped_lock lock(mRefLock); - LLINT_ASSERT(mRefCount != NOT_REF_COUNTED); - LLINT_ASSERT(mRefCount > 0); + llassert_always(mRefCount != NOT_REF_COUNTED); + llassert_always(mRefCount > 0); count = --mRefCount; // CRITICAL SECTION } @@ -104,8 +104,8 @@ inline bool RefCounted::isLastRef() const // CRITICAL SECTION boost::mutex::scoped_lock lock(mRefLock); - LLINT_ASSERT(mRefCount != NOT_REF_COUNTED); - LLINT_ASSERT(mRefCount >= 1); + llassert_always(mRefCount != NOT_REF_COUNTED); + llassert_always(mRefCount >= 1); count = mRefCount; // CRITICAL SECTION } @@ -125,7 +125,7 @@ inline int RefCounted::getRefCount() const inline void RefCounted::noRef() const { boost::mutex::scoped_lock lock(mRefLock); - LLINT_ASSERT(mRefCount <= 1); + llassert_always(mRefCount <= 1); mRefCount = NOT_REF_COUNTED; } diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index b5872606b8..273acae132 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -28,6 +28,7 @@ #include #include +#include namespace LLCore @@ -46,6 +47,16 @@ HttpStatus::operator unsigned long() const } +std::string HttpStatus::toHex() const +{ + std::ostringstream result; + result.width(8); + result.fill('0'); + result << std::hex << operator unsigned long(); + return result.str(); + +} + std::string HttpStatus::toString() const { static const char * llcore_errors[] = @@ -54,7 +65,8 @@ std::string HttpStatus::toString() const "HTTP error reply status", "Services shutting down", "Operation canceled", - "Invalid Content-Range header encountered" + "Invalid Content-Range header encountered", + "Request handle not found" }; static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); @@ -131,6 +143,7 @@ std::string HttpStatus::toString() const default: if (isHttpStatus()) { + // Binary search for the error code and string int bottom(0), top(http_errors_count); while (true) { diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index f81be7103e..c01a5f85d3 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -134,7 +134,10 @@ enum HttpError HE_OP_CANCELED = 3, // Invalid content range header received. - HE_INV_CONTENT_RANGE_HDR = 4 + HE_INV_CONTENT_RANGE_HDR = 4, + + // Request handle not found + HE_HANDLE_NOT_FOUND = 5 }; // end enum HttpError @@ -229,6 +232,9 @@ struct HttpStatus { return operator unsigned long(); } + + /// And to convert to a hex string. + std::string toHex() const; /// Convert status to a string representation. For /// success, returns an empty string. For failure diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 6c62f931ff..a06b859a91 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -31,6 +31,7 @@ #include "_httpservice.h" #include "_httpoperation.h" #include "_httpoprequest.h" +#include "_httpopsetpriority.h" #include "_httpopcancel.h" @@ -162,7 +163,7 @@ HttpStatus HttpRequest::getStatus() const HttpHandle HttpRequest::requestGetByteRange(unsigned int policy_id, - float priority, + unsigned int priority, const std::string & url, size_t offset, size_t len, @@ -190,6 +191,34 @@ HttpHandle HttpRequest::requestGetByteRange(unsigned int policy_id, } +HttpHandle HttpRequest::requestPost(unsigned int policy_id, + unsigned int priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * user_handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpRequest * op = new HttpOpRequest(); + if (! (status = op->setupPost(policy_id, priority, url, body, options, headers))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setHandlers(mReplyQueue, mSelfHandler, user_handler); + mRequestQueue->addOp(op); // transfers refcount + + mLastReqStatus = status; + handle = static_cast(op); + + return handle; +} + + HttpHandle HttpRequest::requestCancel(HttpHandle handle, HttpHandler * user_handler) { HttpStatus status; @@ -222,6 +251,23 @@ HttpHandle HttpRequest::requestNoOp(HttpHandler * user_handler) } +HttpHandle HttpRequest::requestSetPriority(HttpHandle request, unsigned int priority, + HttpHandler * handler) +{ + HttpStatus status; + HttpHandle ret_handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetPriority * op = new HttpOpSetPriority(request, priority); + op->setHandlers(mReplyQueue, mSelfHandler, handler); + mRequestQueue->addOp(op); // transfer refcount as well + + mLastReqStatus = status; + ret_handle = static_cast(op); + + return ret_handle; +} + + HttpStatus HttpRequest::update(long millis) { HttpStatus status; @@ -259,7 +305,7 @@ HttpStatus HttpRequest::createService() { HttpStatus status; - LLINT_ASSERT(! has_inited); + llassert_always(! has_inited); HttpRequestQueue::init(); HttpRequestQueue * rq = HttpRequestQueue::instanceOf(); HttpService::init(rq); @@ -273,7 +319,7 @@ HttpStatus HttpRequest::destroyService() { HttpStatus status; - LLINT_ASSERT(has_inited); + llassert_always(has_inited); HttpService::term(); HttpRequestQueue::term(); has_inited = false; diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 4bbd13a13a..e2ab9be533 100644 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -41,6 +41,7 @@ class HttpService; class HttpOptions; class HttpHeaders; class HttpOperation; +class BufferArray; /// HttpRequest supplies the entry into the HTTP transport /// services in the LLCore libraries. Services provided include: @@ -96,6 +97,10 @@ public: /// Represents a default, catch-all policy class that guarantees /// eventual service for any HTTP request. static const int DEFAULT_POLICY_ID = 0; + + /// Maximum number of policies that may be defined. No policy + /// ID will equal or exceed this value. + static const int POLICY_CLASS_LIMIT = 1; enum EGlobalPolicy { @@ -177,7 +182,7 @@ public: /// @param policy_id Default or user-defined policy class under /// which this request is to be serviced. /// @param priority Standard priority scheme inherited from - /// Indra code base. + /// Indra code base (U32-type scheme). /// @param url /// @param offset /// @param len @@ -190,7 +195,7 @@ public: /// case, @see getStatus() will return more info. /// HttpHandle requestGetByteRange(unsigned int policy_id, - float priority, + unsigned int priority, const std::string & url, size_t offset, size_t len, @@ -199,6 +204,32 @@ public: HttpHandler * handler); + /// + /// @param policy_id Default or user-defined policy class under + /// which this request is to be serviced. + /// @param priority Standard priority scheme inherited from + /// Indra code base. + /// @param url + /// @param body Byte stream to be sent as the body. No + /// further encoding or escaping will be done + /// to the content. + /// @param options (optional) + /// @param headers (optional) + /// @param handler (optional) + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. In the latter + /// case, @see getStatus() will return more info. + /// + HttpHandle requestPost(unsigned int policy_id, + unsigned int priority, + const std::string & url, + BufferArray * body, + HttpOptions * options, + HttpHeaders * headers, + HttpHandler * handler); + + /// Queue a NoOp request. /// The request is queued and serviced by the working thread which /// immediately processes it and returns the request to the reply @@ -235,6 +266,20 @@ public: HttpHandle requestCancel(HttpHandle request, HttpHandler *); + /// Request that a previously-issued request be reprioritized. + /// The status of whether the change itself succeeded arrives + /// via notification. + /// + /// @param request Handle of previously-issued request to + /// be changed. + /// @param priority New priority value. + /// @param handler (optional) + /// @return The handle of the request if successfully + /// queued or LLCORE_HTTP_HANDLE_INVALID if the + /// request could not be queued. + /// + HttpHandle requestSetPriority(HttpHandle request, unsigned int priority, HttpHandler * handler); + /// @} /// @name UtilityMethods diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index a73d90957e..0e9d7d8979 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -370,7 +370,7 @@ void HttpRequestTestObjectType::test<5>() // Issue a GET that can't connect mStatus = HttpStatus(HttpStatus::EXT_CURL_EASY, CURLE_COULDNT_CONNECT); HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, - 0.0f, + 0U, "http://127.0.0.1:2/nothing/here", 0, 0, diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 17c68f7c22..381364b5c3 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -51,6 +51,7 @@ #include "llviewerstats.h" #include "llviewerassetstats.h" #include "llworld.h" +#include "llsdserialize.h" #include "httprequest.h" #include "httphandler.h" @@ -748,7 +749,7 @@ void LLTextureFetchWorker::setImagePriority(F32 priority) { mImagePriority = priority; calcWorkPriority(); - U32 work_priority = mWorkPriority | LLWorkerThread::PRIORITY_LOW; + U32 work_priority = mWorkPriority | (getPriority() & LLWorkerThread::PRIORITY_HIGHBITS); setPriority(work_priority); } } @@ -855,7 +856,6 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == LOAD_FROM_TEXTURE_CACHE) { - setPriority(0); // Set priority first since Responder may change it if (mCacheReadHandle == LLTextureCache::nullHandle()) { U32 cache_priority = mWorkPriority; @@ -871,6 +871,8 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mUrl.compare(0, 7, "file://") == 0) { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + // read file from local disk std::string filename = mUrl.substr(7, std::string::npos); CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); @@ -879,6 +881,8 @@ bool LLTextureFetchWorker::doWork(S32 param) } else if (mUrl.empty()) { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it + CacheReadResponder* responder = new CacheReadResponder(mFetcher, mID, mFormattedImage); mCacheReadHandle = mFetcher->mTextureCache->readFromCache(mID, cache_priority, offset, size, responder); @@ -890,12 +894,12 @@ bool LLTextureFetchWorker::doWork(S32 param) // *TODO:?remove this warning llwarns << "Unknown URL Type: " << mUrl << llendl; } - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); mState = SEND_HTTP_REQ; } else { - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); mState = LOAD_FROM_NETWORK; } } @@ -907,7 +911,6 @@ bool LLTextureFetchWorker::doWork(S32 param) { mCacheReadHandle = LLTextureCache::nullHandle(); mState = CACHE_POST; - setPriority(LLWorkerThread::PRIORITY_HIGH); // fall through } else @@ -931,7 +934,6 @@ bool LLTextureFetchWorker::doWork(S32 param) llassert_always(mFormattedImage->getDataSize() > 0); mLoadedDiscard = mDesiredDiscard; mState = DECODE_IMAGE; - setPriority(LLWorkerThread::PRIORITY_HIGH); mWriteToCacheState = NOT_WRITE ; LL_DEBUGS("Texture") << mID << ": Cached. Bytes: " << mFormattedImage->getDataSize() << " Size: " << llformat("%dx%d",mFormattedImage->getWidth(),mFormattedImage->getHeight()) @@ -949,7 +951,6 @@ bool LLTextureFetchWorker::doWork(S32 param) else { LL_DEBUGS("Texture") << mID << ": Not in Cache" << LL_ENDL; - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = LOAD_FROM_NETWORK; } // fall through @@ -960,7 +961,6 @@ bool LLTextureFetchWorker::doWork(S32 param) { static LLCachedControl use_http(gSavedSettings,"ImagePipelineUseHTTP"); - setPriority(0); // if (mHost != LLHost::invalid) get_url = false; if ( use_http && mCanUseHTTP && mUrl.empty())//get http url. { @@ -993,7 +993,7 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mCanUseHTTP && !mUrl.empty()) { mState = SEND_HTTP_REQ; - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); if(mWriteToCacheState != NOT_WRITE) { mWriteToCacheState = CAN_WRITE ; @@ -1010,6 +1010,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mSentRequest = QUEUED; mFetcher->addToNetworkQueue(this); recordTextureStart(false); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; } @@ -1027,7 +1028,6 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == LOAD_FROM_SIMULATOR) { - setPriority(0); if (mFormattedImage.isNull()) { mFormattedImage = new LLImageJ2C; @@ -1042,7 +1042,7 @@ bool LLTextureFetchWorker::doWork(S32 param) // llwarns << "processSimulatorPackets() failed to load buffer" << llendl; return true; // failed } - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); mState = DECODE_IMAGE; mWriteToCacheState = SHOULD_WRITE; recordTextureDone(false); @@ -1050,6 +1050,7 @@ bool LLTextureFetchWorker::doWork(S32 param) else { mFetcher->addToNetworkQueue(this); // failsafe + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); recordTextureStart(false); } return false; @@ -1057,7 +1058,6 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == SEND_HTTP_REQ) { - setPriority(0); if(mCanUseHTTP) { //NOTE: @@ -1065,11 +1065,13 @@ bool LLTextureFetchWorker::doWork(S32 param) //1, not openning too many file descriptors at the same time; //2, control the traffic of http so udp gets bandwidth. // - static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8 ; - if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) - { - return false ; //wait. - } + static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8; + // *FIXME: For the moment, allow everything to transition into HTTP + // and have the new library order and throttle. + //if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) + //{ + //return false ; //wait. + //} mFetcher->removeFromNetworkQueue(this, false); @@ -1083,7 +1085,6 @@ bool LLTextureFetchWorker::doWork(S32 param) { // We already have all the data, just decode it mLoadedDiscard = mFormattedImage->getDiscardLevel(); - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; return false; } @@ -1135,7 +1136,7 @@ bool LLTextureFetchWorker::doWork(S32 param) mHttpActive = true; mFetcher->addToHTTPQueue(mID); recordTextureStart(true); - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); mState = WAIT_HTTP_REQ; // fall through @@ -1148,7 +1149,6 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WAIT_HTTP_REQ) { - setPriority(0); if (mLoaded) { S32 cur_size = mFormattedImage.notNull() ? mFormattedImage->getDataSize() : 0; @@ -1165,7 +1165,7 @@ bool LLTextureFetchWorker::doWork(S32 param) { mState = INIT; mCanUseHTTP = false; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return false; } } @@ -1184,7 +1184,8 @@ bool LLTextureFetchWorker::doWork(S32 param) max_attempts = HTTP_MAX_RETRY_COUNT + 1; ++mHTTPFailCount; llinfos << "HTTP GET failed for: " << mUrl - << " Status: " << mGetStatus.toULong() << " Reason: '" << mGetReason << "'" + << " Status: " << mGetStatus.toHex() + << " Reason: '" << mGetReason << "'" << " Attempt:" << mHTTPFailCount+1 << "/" << max_attempts << llendl; } @@ -1194,14 +1195,12 @@ bool LLTextureFetchWorker::doWork(S32 param) { // Use available data mLoadedDiscard = mFormattedImage->getDiscardLevel(); - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; return false; } else { resetFormattedData(); - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; return true; // failed } @@ -1223,7 +1222,6 @@ bool LLTextureFetchWorker::doWork(S32 param) } // abort. - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; return true; } @@ -1268,29 +1266,30 @@ bool LLTextureFetchWorker::doWork(S32 param) mHttpBufferArray = NULL; mLoadedDiscard = mRequestedDiscard; - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DECODE_IMAGE; if (mWriteToCacheState != NOT_WRITE) { mWriteToCacheState = SHOULD_WRITE ; } + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return false; } else { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return false; } } if (mState == DECODE_IMAGE) { - setPriority(0); static LLCachedControl textures_decode_disabled(gSavedSettings,"TextureDecodeDisabled"); + + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it if (textures_decode_disabled) { // for debug use, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } @@ -1298,7 +1297,6 @@ bool LLTextureFetchWorker::doWork(S32 param) { // We aborted, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } @@ -1308,7 +1306,6 @@ bool LLTextureFetchWorker::doWork(S32 param) //abort, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } if (mLoadedDiscard < 0) @@ -1317,7 +1314,6 @@ bool LLTextureFetchWorker::doWork(S32 param) //abort, don't decode mState = DONE; - setPriority(LLWorkerThread::PRIORITY_HIGH); return true; } @@ -1337,7 +1333,6 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DECODE_IMAGE_UPDATE) { - setPriority(0); if (mDecoded) { if (mDecodedDiscard < 0) @@ -1350,14 +1345,13 @@ bool LLTextureFetchWorker::doWork(S32 param) llassert_always(mDecodeHandle == 0); mFormattedImage = NULL; ++mRetryAttempt; - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); mState = INIT; return false; } else { // llwarns << "UNABLE TO LOAD TEXTURE: " << mID << " RETRIES: " << mRetryAttempt << llendl; - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; // failed } } @@ -1366,7 +1360,7 @@ bool LLTextureFetchWorker::doWork(S32 param) llassert_always(mRawImage.notNull()); LL_DEBUGS("Texture") << mID << ": Decoded. Discard: " << mDecodedDiscard << " Raw Image: " << llformat("%dx%d",mRawImage->getWidth(),mRawImage->getHeight()) << LL_ENDL; - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); mState = WRITE_TO_CACHE; } // fall through @@ -1379,12 +1373,10 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WRITE_TO_CACHE) { - setPriority(0); if (mWriteToCacheState != SHOULD_WRITE || mFormattedImage.isNull()) { // If we're in a local cache or we didn't actually receive any new data, // or we failed to load anything, skip - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; return false; } @@ -1404,7 +1396,6 @@ bool LLTextureFetchWorker::doWork(S32 param) setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); // Set priority first since Responder may change it U32 cache_priority = mWorkPriority; mWritten = FALSE; - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = WAIT_ON_WRITE; CacheWriteResponder* responder = new CacheWriteResponder(mFetcher, mID); mCacheWriteHandle = mFetcher->mTextureCache->writeToCache(mID, cache_priority, @@ -1415,10 +1406,8 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == WAIT_ON_WRITE) { - setPriority(0); if (writeToCacheComplete()) { - setPriority(LLWorkerThread::PRIORITY_HIGH); mState = DONE; // fall through } @@ -1437,15 +1426,16 @@ bool LLTextureFetchWorker::doWork(S32 param) if (mState == DONE) { - setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); if (mDecodedDiscard >= 0 && mDesiredDiscard < mDecodedDiscard) { // More data was requested, return to INIT mState = INIT; + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return false; } else { + setPriority(LLWorkerThread::PRIORITY_LOW | mWorkPriority); return true; } } @@ -1477,7 +1467,8 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe LLCore::HttpStatus status(response->getStatus()); lldebugs << "HTTP COMPLETE: " << mID - << " status: " << status.toULong() << " '" << status.toString() << "'" + << " status: " << status.toHex() + << " '" << status.toString() << "'" << llendl; unsigned int offset(0), length(0); response->getRange(&offset, &length); @@ -1492,7 +1483,8 @@ void LLTextureFetchWorker::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRe success = false; std::string reason(status.toString()); setGetStatus(status, reason); - llwarns << "CURL GET FAILED, status:" << status.toULong() << " reason:" << reason << llendl; + llwarns << "CURL GET FAILED, status: " << status.toHex() + << " reason: " << reason << llendl; } else { @@ -1727,7 +1719,7 @@ S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, mRequestedSize = -1; // error } mLoaded = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); return data_size ; } @@ -1756,7 +1748,7 @@ void LLTextureFetchWorker::callbackCacheRead(bool success, LLImageFormatted* ima } } mLoaded = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); } void LLTextureFetchWorker::callbackCacheWrite(bool success) @@ -1768,7 +1760,7 @@ void LLTextureFetchWorker::callbackCacheWrite(bool success) return; } mWritten = TRUE; - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); } ////////////////////////////////////////////////////////////////////////////// @@ -1806,7 +1798,7 @@ void LLTextureFetchWorker::callbackDecoded(bool success, LLImageRaw* raw, LLImag } mDecoded = TRUE; // llinfos << mID << " : DECODE COMPLETE " << llendl; - setPriority(LLWorkerThread::PRIORITY_HIGH); + setPriority(LLWorkerThread::PRIORITY_HIGH | mWorkPriority); } ////////////////////////////////////////////////////////////////////////////// @@ -1883,7 +1875,6 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpOptions(NULL), mHttpHeaders(NULL) { - mCurlPOSTRequestCount = 0; mMaxBandwidth = gSavedSettings.getF32("ThrottleBandwidthKBPS"); mTextureInfo.setUpLogging(gSavedSettings.getBOOL("LogTextureDownloadsToViewerLog"), gSavedSettings.getBOOL("LogTextureDownloadsToSimulator"), gSavedSettings.getU32("TextureLoggingThreshold")); @@ -2253,7 +2244,6 @@ S32 LLTextureFetch::getPending() LLMutexLock lock(&mQueueMutex); res = mRequestQueue.size(); - res += mCurlPOSTRequestCount; res += mCommands.size(); } unlockData(); @@ -2279,10 +2269,7 @@ bool LLTextureFetch::runCondition() have_no_commands = mCommands.empty(); } - bool have_no_curl_requests(0 == mCurlPOSTRequestCount); - return ! (have_no_commands - && have_no_curl_requests && (mRequestQueue.empty() && mIdleThread)); // From base class } @@ -2690,7 +2677,7 @@ bool LLTextureFetch::receiveImageHeader(const LLHost& host, const LLUUID& id, U8 llassert_always(totalbytes > 0); llassert_always(data_size == FIRST_PACKET_SIZE || data_size == worker->mFileSize); res = worker->insertPacket(0, data, data_size); - worker->setPriority(LLWorkerThread::PRIORITY_HIGH); + worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; worker->unlockWorkMutex(); return res; @@ -2734,7 +2721,7 @@ bool LLTextureFetch::receiveImagePacket(const LLHost& host, const LLUUID& id, U1 if ((worker->mState == LLTextureFetchWorker::LOAD_FROM_SIMULATOR) || (worker->mState == LLTextureFetchWorker::LOAD_FROM_NETWORK)) { - worker->setPriority(LLWorkerThread::PRIORITY_HIGH); + worker->setPriority(LLWorkerThread::PRIORITY_HIGH | worker->mWorkPriority); worker->mState = LLTextureFetchWorker::LOAD_FROM_SIMULATOR; } else @@ -2930,6 +2917,35 @@ void LLTextureFetch::cmdDoWork() namespace { + +// Example of a simple notification handler for metrics +// delivery notification. Earlier versions of the code used +// a Responder that tried harder to detect delivery breaks +// but it really isn't that important. If someone wants to +// revisit that effort, here is a place to start. +class AssetReportHandler : public LLCore::HttpHandler +{ +public: + virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response) + { + LLCore::HttpStatus status(response->getStatus()); + + if (status) + { + LL_WARNS("Texture") << "Successfully delivered asset metrics to grid." + << LL_ENDL; + } + else + { + LL_WARNS("Texture") << "Error delivering asset metrics to grid. Reason: " + << status.toString() << LL_ENDL; + } + } +}; // end class AssetReportHandler + +AssetReportHandler stats_handler; + + /** * Implements the 'Set Region' command. * @@ -2960,73 +2976,9 @@ TFReqSendMetrics::~TFReqSendMetrics() bool TFReqSendMetrics::doWork(LLTextureFetch * fetcher) { - /* - * HTTP POST responder. Doesn't do much but tries to - * detect simple breaks in recording the metrics stream. - * - * The 'volatile' modifiers don't indicate signals, - * mmap'd memory or threads, really. They indicate that - * the referenced data is part of a pseudo-closure for - * this responder rather than being required for correct - * operation. - * - * We don't try very hard with the POST request. We give - * it one shot and that's more-or-less it. With a proper - * refactoring of the LLQueuedThread usage, these POSTs - * could be put in a request object and made more reliable. - */ - class lcl_responder : public LLCurl::Responder - { - public: - lcl_responder(LLTextureFetch * fetcher, - S32 expected_sequence, - volatile const S32 & live_sequence, - volatile bool & reporting_break, - volatile bool & reporting_started) - : LLCurl::Responder(), - mFetcher(fetcher), - mExpectedSequence(expected_sequence), - mLiveSequence(live_sequence), - mReportingBreak(reporting_break), - mReportingStarted(reporting_started) - { - mFetcher->incrCurlPOSTCount(); - } - - ~lcl_responder() - { - mFetcher->decrCurlPOSTCount(); - } - - // virtual - void error(U32 status_num, const std::string & reason) - { - if (mLiveSequence == mExpectedSequence) - { - mReportingBreak = true; - } - LL_WARNS("Texture") << "Break in metrics stream due to POST failure to metrics collection service. Reason: " - << reason << LL_ENDL; - } - - // virtual - void result(const LLSD & content) - { - if (mLiveSequence == mExpectedSequence) - { - mReportingBreak = false; - mReportingStarted = true; - } - } - - private: - LLTextureFetch * mFetcher; - S32 mExpectedSequence; - volatile const S32 & mLiveSequence; - volatile bool & mReportingBreak; - volatile bool & mReportingStarted; - - }; // class lcl_responder + static const U32 report_priority(LLWorkerThread::PRIORITY_LOW); + static const int report_policy_class(LLCore::HttpRequest::DEFAULT_POLICY_ID); + static LLCore::HttpHandler * const handler(fetcher->isQAMode() || true ? &stats_handler : NULL); if (! gViewerAssetStatsThread1) return true; @@ -3054,24 +3006,37 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) // Update sequence number if (S32_MAX == ++report_sequence) report_sequence = 0; - + reporting_started = true; + // Limit the size of the stats report if necessary. merged_llsd["truncated"] = truncate_viewer_metrics(10, merged_llsd); if (! mCapsURL.empty()) { - LLCurlRequest::headers_t headers; -#if 0 - // *FIXME: Going to need a post op after all... - fetcher->getCurlRequest().post(mCapsURL, - headers, - merged_llsd, - new lcl_responder(fetcher, - report_sequence, - report_sequence, - LLTextureFetch::svMetricsDataBreak, - reporting_started)); -#endif + // *FIXME: This mess to get an llsd into a string though + // it's actually no worse than what we currently do... + std::stringstream body; + LLSDSerialize::toXML(merged_llsd, body); + std::string body_str(body.str()); + body.clear(); + + LLCore::HttpHeaders * headers = new LLCore::HttpHeaders; + headers->mHeaders.push_back("Content-Type: application/llsd+xml"); + + LLCore::BufferArray * ba = new LLCore::BufferArray; + ba->append(body_str.c_str(), body_str.length()); + body_str.clear(); + + fetcher->getHttpRequest().requestPost(report_policy_class, + report_priority, + mCapsURL, + ba, + NULL, + headers, + handler); + ba->release(); + headers->release(); + LLTextureFetch::svMetricsDataBreak = false; } else { @@ -3079,7 +3044,7 @@ TFReqSendMetrics::doWork(LLTextureFetch * fetcher) } // In QA mode, Metrics submode, log the result for ease of testing - if (fetcher->isQAMode()) + if (fetcher->isQAMode() || true) { LL_INFOS("Textures") << merged_llsd << LL_ENDL; } diff --git a/indra/newview/lltexturefetch.h b/indra/newview/lltexturefetch.h index 402b198246..cfea3aad9d 100644 --- a/indra/newview/lltexturefetch.h +++ b/indra/newview/lltexturefetch.h @@ -39,7 +39,6 @@ class LLViewerTexture; class LLTextureFetchWorker; -class HTTPGetResponder; class LLTextureCache; class LLImageDecodeThread; class LLHost; @@ -49,7 +48,6 @@ class LLViewerAssetStats; class LLTextureFetch : public LLWorkerThread { friend class LLTextureFetchWorker; - friend class HTTPGetResponder; public: LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* imagedecodethread, bool threaded, bool qa_mode); @@ -90,8 +88,6 @@ public: LLTextureFetchWorker* getWorker(const LLUUID& id); LLTextureFetchWorker* getWorkerAfterLock(const LLUUID& id); - LLTextureInfo* getTextureInfo() { return &mTextureInfo; } - // Commands available to other threads to control metrics gathering operations. void commandSetRegion(U64 region_handle); void commandSendMetrics(const std::string & caps_url, @@ -104,10 +100,6 @@ public: bool isQAMode() const { return mQAMode; } - // Curl POST counter maintenance - inline void incrCurlPOSTCount() { mCurlPOSTRequestCount++; } - inline void decrCurlPOSTCount() { mCurlPOSTRequestCount--; } - protected: void addToNetworkQueue(LLTextureFetchWorker* worker); void removeFromNetworkQueue(LLTextureFetchWorker* worker, bool cancel); @@ -199,13 +191,6 @@ private: // If true, modifies some behaviors that help with QA tasks. const bool mQAMode; - // Count of POST requests outstanding. We maintain the count - // indirectly in the CURL request responder's ctor and dtor and - // use it when determining whether or not to sleep the thread. Can't - // use the LLCurl module's request counter as it isn't thread compatible. - // *NOTE: Don't mix Atomic and static, apr_initialize must be called first. - LLAtomic32 mCurlPOSTRequestCount; - // Interfaces and objects into the core http library used // to make our HTTP requests. These replace the various // LLCurl interfaces used in the past. -- cgit v1.3 From fb5a29c069d27611b6328fbc313382ef0914ffe9 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 1 Jun 2012 23:06:24 +0000 Subject: Platform fixups Linux: unused variables, make error strings constant. --- indra/llcommon/llsdserialize.cpp | 5 ++++- indra/llcommon/tests/bitpack_test.cpp | 1 + indra/llcommon/tests/reflection_test.cpp | 2 +- indra/llcorehttp/httpcommon.cpp | 2 +- indra/llmessage/llhttpassetstorage.cpp | 2 +- indra/newview/lltexturefetch.cpp | 10 +++++----- 6 files changed, 13 insertions(+), 9 deletions(-) (limited to 'indra/llcorehttp/httpcommon.cpp') diff --git a/indra/llcommon/llsdserialize.cpp b/indra/llcommon/llsdserialize.cpp index bf62600514..f29c96f17c 100644 --- a/indra/llcommon/llsdserialize.cpp +++ b/indra/llcommon/llsdserialize.cpp @@ -1447,9 +1447,12 @@ S32 LLSDBinaryFormatter::format(const LLSD& data, std::ostream& ostr, U32 option } case LLSD::TypeUUID: + { ostr.put('u'); - ostr.write((const char*)(&(data.asUUID().mData)), UUID_BYTES); + LLSD::UUID value = data.asUUID(); + ostr.write((const char*)(&value.mData), UUID_BYTES); break; + } case LLSD::TypeString: ostr.put('s'); diff --git a/indra/llcommon/tests/bitpack_test.cpp b/indra/llcommon/tests/bitpack_test.cpp index 05289881d0..4c3bc674af 100644 --- a/indra/llcommon/tests/bitpack_test.cpp +++ b/indra/llcommon/tests/bitpack_test.cpp @@ -95,6 +95,7 @@ namespace tut ensure("bitPack: individual unpack: 5", unpackbuffer[0] == (U8) str[5]); unpack_bufsize = bitunpack.bitUnpack(unpackbuffer, 8*4); // Life ensure_memory_matches("bitPack: 4 bytes unpack:", unpackbuffer, 4, str+6, 4); + ensure("keep compiler quiet", unpack_bufsize == unpack_bufsize); } // U32 packing diff --git a/indra/llcommon/tests/reflection_test.cpp b/indra/llcommon/tests/reflection_test.cpp index 59491cd1fe..8980ebb1f1 100644 --- a/indra/llcommon/tests/reflection_test.cpp +++ b/indra/llcommon/tests/reflection_test.cpp @@ -207,7 +207,7 @@ namespace tut const LLReflective* reflective = property->get(aggregated_data); // Wrong reflective type, should throw exception. // useless op to get rid of compiler warning. - reflective = NULL; + reflective = reflective; } catch(...) { diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 273acae132..a01182cf23 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -73,7 +73,7 @@ std::string HttpStatus::toString() const static const struct { type_enum_t mCode; - char * mText; + const char * mText; } http_errors[] = { diff --git a/indra/llmessage/llhttpassetstorage.cpp b/indra/llmessage/llhttpassetstorage.cpp index 612d765969..d6ed08055e 100644 --- a/indra/llmessage/llhttpassetstorage.cpp +++ b/indra/llmessage/llhttpassetstorage.cpp @@ -749,7 +749,7 @@ LLAssetRequest* LLHTTPAssetStorage::findNextRequest(LLAssetStorage::request_list request_list_t::iterator pending_iter = pending.begin(); request_list_t::iterator pending_end = pending.end(); // Loop over all pending requests until we miss finding it in the running list. - for (; pending_iter != pending.end(); ++pending_iter) + for (; pending_iter != pending_end; ++pending_iter) { LLAssetRequest* req = *pending_iter; // Look for this pending request in the running list. diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 381364b5c3..daad463e0d 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -1065,13 +1065,13 @@ bool LLTextureFetchWorker::doWork(S32 param) //1, not openning too many file descriptors at the same time; //2, control the traffic of http so udp gets bandwidth. // - static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8; + // static const S32 MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE = 8; // *FIXME: For the moment, allow everything to transition into HTTP // and have the new library order and throttle. - //if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) - //{ - //return false ; //wait. - //} + // if(mFetcher->getNumHTTPRequests() > MAX_NUM_OF_HTTP_REQUESTS_IN_QUEUE) + // { + // return false ; //wait. + // } mFetcher->removeFromNetworkQueue(this, false); -- cgit v1.3 From 05af16a23abe37210e0b880aa27387d8994419dd Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 6 Jun 2012 13:52:38 -0400 Subject: Policy + caching fixes + https support + POST working Implemented first global policy definitions to support SSL CA certificate configuration to support https: operations. Fixed HTTP 206 status handling to match what is currently being done by grid services and to lay a foundation for fixes that will be a response to ER-1824. More libcurl CURLOPT options set on easy handles to do peer verification in the traditional way. HTTP POST working and now reporting asset metrics back to grid for the viewer's asset system. This uses LLSD so that is also showing as compatible with the new library. --- indra/llcorehttp/CMakeLists.txt | 2 ++ indra/llcorehttp/_httplibcurl.cpp | 1 + indra/llcorehttp/_httpoprequest.cpp | 45 ++++++++++++++++++++++++++++++++----- indra/llcorehttp/_httppolicy.cpp | 4 ++-- indra/llcorehttp/_httppolicy.h | 10 +++++++++ indra/llcorehttp/_httpservice.h | 8 +++---- indra/llcorehttp/httpcommon.cpp | 5 ++++- indra/llcorehttp/httpcommon.h | 11 ++++++++- indra/llcorehttp/httprequest.cpp | 13 +++++++++-- indra/llcorehttp/httprequest.h | 16 +++++++------ indra/newview/llappviewer.cpp | 12 +++++++++- indra/newview/lltexturefetch.cpp | 37 +++++++++++++++++------------- 12 files changed, 126 insertions(+), 38 deletions(-) (limited to 'indra/llcorehttp/httpcommon.cpp') diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt index 9a073eb850..3fda524ddf 100644 --- a/indra/llcorehttp/CMakeLists.txt +++ b/indra/llcorehttp/CMakeLists.txt @@ -33,6 +33,7 @@ set(llcorehttp_SOURCE_FILES _httpoprequest.cpp _httpopsetpriority.cpp _httppolicy.cpp + _httppolicyglobal.cpp _httpreplyqueue.cpp _httprequestqueue.cpp _httpservice.cpp @@ -55,6 +56,7 @@ set(llcorehttp_HEADER_FILES _httpoprequest.h _httpopsetpriority.h _httppolicy.h + _httppolicyglobal.h _httpreadyqueue.h _httpreplyqueue.h _httprequestqueue.h diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 704f9baac9..5272c391e8 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -27,6 +27,7 @@ #include "_httplibcurl.h" #include "httpheaders.h" +#include "bufferarray.h" #include "_httpoprequest.h" diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index f52ff5a44c..4bdc4a5257 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -40,8 +40,10 @@ #include "_httpreplyqueue.h" #include "_httpservice.h" #include "_httppolicy.h" +#include "_httppolicyglobal.h" #include "_httplibcurl.h" +#include "llhttpstatuscodes.h" namespace { @@ -153,14 +155,14 @@ HttpOpRequest::~HttpOpRequest() void HttpOpRequest::stageFromRequest(HttpService * service) { addRef(); - service->getPolicy()->addOp(this); // transfers refcount + service->getPolicy().addOp(this); // transfers refcount } void HttpOpRequest::stageFromReady(HttpService * service) { addRef(); - service->getTransport()->addOp(this); // transfers refcount + service->getTransport().addOp(this); // transfers refcount } @@ -195,6 +197,8 @@ void HttpOpRequest::stageFromActive(HttpService * service) void HttpOpRequest::visitNotifier(HttpRequest * request) { + static const HttpStatus partial_content(HTTP_PARTIAL_CONTENT, HE_SUCCESS); + if (mLibraryHandler) { HttpResponse * response = new HttpResponse(); @@ -208,9 +212,15 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) offset = mReplyOffset; length = mReplyLength; } - else if (mReplyBody) + else if (mReplyBody && partial_content == mStatus) { - // Provide implicit offset/length from request/response + // Legacy grid services did not provide a 'Content-Range' + // header in responses to full- or partly-satisfyiable + // 'Range' requests. For these, we have to hope that + // the data starts where requested and the length is simply + // whatever we received. A bit of sanity could be provided + // by overlapping ranged requests and verifying that the + // overlap matches. offset = mReqOffset; length = mReplyBody->size(); } @@ -306,6 +316,9 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *FIXME: better error handling later HttpStatus status; + // Get policy options + HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions()); + mCurlHandle = curl_easy_init(); // curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, 30); @@ -322,21 +335,40 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10); + curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10); // *FIXME: parameterize this later curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle); curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); curl_easy_setopt(mCurlHandle, CURLOPT_READDATA, mCurlHandle); + curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); + std::string opt_value; + if (policy.get(HttpRequest::GP_CA_PATH, opt_value)) + { + curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value.c_str()); + } + if (policy.get(HttpRequest::GP_CA_FILE, opt_value)) + { + curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value.c_str()); + } + if (policy.get(HttpRequest::GP_HTTP_PROXY, opt_value)) + { + curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value.c_str()); + } + switch (mReqMethod) { case HOR_GET: curl_easy_setopt(mCurlHandle, CURLOPT_HTTPGET, 1); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); break; case HOR_POST: { curl_easy_setopt(mCurlHandle, CURLOPT_POST, 1); + curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, ""); long data_size(0); if (mReqBody) { @@ -358,8 +390,11 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) data_size = mReqBody->size(); } curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); + curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); mCurlHeaders = curl_slist_append(mCurlHeaders, "Transfer-Encoding: chunked"); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); + mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); } break; diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 1d28f23d56..51f5e487dc 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -76,12 +76,12 @@ void HttpPolicy::addOp(HttpOpRequest * op) HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); - HttpLibcurl * pTransport(mService->getTransport()); + HttpLibcurl & transport(mService->getTransport()); for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) { HttpReadyQueue & readyq(mReadyQueue[policy_class]); - int active(pTransport->getActiveCountInClass(policy_class)); + int active(transport.getActiveCountInClass(policy_class)); int needed(8 - active); if (needed > 0 && mReadyInClass[policy_class] > 0) diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 2bc03c531f..425079ec63 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -31,6 +31,7 @@ #include "httprequest.h" #include "_httpservice.h" #include "_httpreadyqueue.h" +#include "_httppolicyglobal.h" namespace LLCore @@ -68,11 +69,20 @@ public: // Shadows HttpService's method bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); + + // Get pointer to global policy options. Caller is expected + // to do context checks like no setting once running. + HttpPolicyGlobal & getGlobalOptions() + { + return mGlobalOptions; + } + protected: int mReadyInClass[HttpRequest::POLICY_CLASS_LIMIT]; HttpReadyQueue mReadyQueue[HttpRequest::POLICY_CLASS_LIMIT]; HttpService * mService; // Naked pointer, not refcounted, not owner + HttpPolicyGlobal mGlobalOptions; }; // end class HttpPolicy diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index 095316c8a7..748354a8e4 100644 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -148,14 +148,14 @@ public: /// Threading: callable by worker thread. bool changePriority(HttpHandle handle, HttpRequest::priority_t priority); - HttpPolicy * getPolicy() + HttpPolicy & getPolicy() { - return mPolicy; + return *mPolicy; } - HttpLibcurl * getTransport() + HttpLibcurl & getTransport() { - return mTransport; + return *mTransport; } protected: diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index a01182cf23..9f17b5c842 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -66,7 +66,10 @@ std::string HttpStatus::toString() const "Services shutting down", "Operation canceled", "Invalid Content-Range header encountered", - "Request handle not found" + "Request handle not found", + "Invalid datatype for argument or option", + "Option has not been explicitly set", + "Option is not dynamic and must be set early" }; static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index c01a5f85d3..fd2661b700 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -137,7 +137,16 @@ enum HttpError HE_INV_CONTENT_RANGE_HDR = 4, // Request handle not found - HE_HANDLE_NOT_FOUND = 5 + HE_HANDLE_NOT_FOUND = 5, + + // Invalid datatype for option/setting + HE_INVALID_ARG = 6, + + // Option hasn't been explicitly set + HE_OPT_NOT_SET = 7, + + // Option not dynamic, must be set during init phase + HE_OPT_NOT_DYNAMIC = 8 }; // end enum HttpError diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 0e512d97ed..baa0fe1a84 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -29,6 +29,7 @@ #include "_httprequestqueue.h" #include "_httpreplyqueue.h" #include "_httpservice.h" +#include "_httppolicy.h" #include "_httpoperation.h" #include "_httpoprequest.h" #include "_httpopsetpriority.h" @@ -127,9 +128,17 @@ HttpRequest::~HttpRequest() HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value) { - HttpStatus status; + // *FIXME: Fail if thread is running. - return status; + return HttpService::instanceOf()->getPolicy().getGlobalOptions().set(opt, value); +} + + +HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value) +{ + // *FIXME: Fail if thread is running. + + return HttpService::instanceOf()->getPolicy().getGlobalOptions().set(opt, value); } diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 57d2da245b..3592d5c6a3 100644 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -111,7 +111,10 @@ public: /// Maximum number of connections the library will use to /// perform operations. This is somewhat soft as the underlying /// transport will cache some connections (up to 5). - GLOBAL_CONNECTION_LIMIT + GP_CONNECTION_LIMIT, ///< Takes long giving number of connections + GP_CA_PATH, ///< System path/directory where SSL certs are stored. + GP_CA_FILE, ///< System path/file containing certs. + GP_HTTP_PROXY ///< String giving host/port to use for HTTP proxy }; /// Set a parameter on a global policy option. Calls @@ -122,6 +125,7 @@ public: /// @param value Desired value of option. /// @return Standard status code. HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value); + HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value); /// Create a new policy class into which requests can be made. /// @@ -134,15 +138,15 @@ public: enum EClassPolicy { /// Limits the number of connections used for the class. - CLASS_CONNECTION_LIMIT, + CP_CONNECTION_LIMIT, /// Limits the number of connections used for a single /// literal address/port pair within the class. - PER_HOST_CONNECTION_LIMIT, + CP_PER_HOST_CONNECTION_LIMIT, /// Suitable requests are allowed to pipeline on their /// connections when they ask for it. - ENABLE_PIPELINING + CP_ENABLE_PIPELINING }; /// Set a parameter on a class-based policy option. Calls @@ -153,9 +157,7 @@ public: /// @param opt Enum of option to be set. /// @param value Desired value of option. /// @return Standard status code. - HttpStatus setPolicyClassOption(policy_t policy_id, - EClassPolicy opt, - long value); + HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); /// @} diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index 8e6deb9cce..7a44415fba 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5356,6 +5356,17 @@ void CoreHttp::init() << LL_ENDL; } + mRequest = new LLCore::HttpRequest; + + status = mRequest->setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, + gDirUtilp->getCAFile()); + if (! status) + { + LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " + << status.toString() + << LL_ENDL; + } + status = LLCore::HttpRequest::startThread(); if (! status) { @@ -5364,7 +5375,6 @@ void CoreHttp::init() << LL_ENDL; } - mRequest = new LLCore::HttpRequest; } diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 34fb21798f..f9294b4cd1 100644 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -816,14 +816,6 @@ bool LLTextureFetchWorker::doWork(S32 param) mFetchTimer.reset(); } - static LLUUID last_id; - if (mID != last_id) - { - // LL_WARNS("Texture") << "DOWORK SWITCH: " << last_id << " to: " << mID - // << LL_ENDL; - last_id = mID; - } - if (mState == INIT) { mRawImage = NULL ; @@ -1109,10 +1101,6 @@ bool LLTextureFetchWorker::doWork(S32 param) << " Bytes: " << mRequestedSize << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth << LL_ENDL; -// LL_WARNS("Texture") << "HTTP GET: " << mID << " Offset: " << mRequestedOffset -// << " Bytes: " << mRequestedSize -// << " Bandwidth(kbps): " << mFetcher->getTextureBandwidth() << "/" << mFetcher->mMaxBandwidth -// << LL_ENDL; // Will call callbackHttpGet when curl request completes // *FIXME: enable redirection follow @@ -1241,7 +1229,7 @@ bool LLTextureFetchWorker::doWork(S32 param) } } - if (mHaveAllData && mRequestedDiscard == 0) //the image file is fully loaded. + if (mHaveAllData /* && mRequestedDiscard == 0*/) //the image file is fully loaded. { mFileSize = total_size; } @@ -1692,13 +1680,32 @@ S32 LLTextureFetchWorker::callbackHttpGet(LLCore::HttpResponse * response, body->addRef(); mHttpBufferArray = body; - if (data_size < mRequestedSize && mRequestedDiscard == 0) + if (! partial) + { + // Response indicates this is the entire asset regardless + // of our asking for a byte range. Mark it so and drop + // any partial data we might have so that the current + // response body becomes the entire dataset. + if (data_size <= mRequestedOffset) + { + LL_WARNS("Texture") << "Fetched entire texture " << mID + << " when it was expected to be marked complete. mImageSize: " + << mFileSize << " datasize: " << mFormattedImage->getDataSize() + << LL_ENDL; + } + mHaveAllData = TRUE; + llassert_always(mDecodeHandle == 0); + mFormattedImage = NULL; // discard any previous data we had + } + else if (data_size < mRequestedSize && mRequestedDiscard == 0) { + // *FIXME: I think we can treat this as complete regardless + // of requested discard level. Revisit this... mHaveAllData = TRUE; } else if (data_size > mRequestedSize) { - // *TODO: This shouldn't be happening any more + // *TODO: This shouldn't be happening any more (REALLY don't expect this anymore) llwarns << "data_size = " << data_size << " > requested: " << mRequestedSize << llendl; mHaveAllData = TRUE; llassert_always(mDecodeHandle == 0); -- cgit v1.3 From a50944e078b98435685f04eda0ba93e95d4c61f2 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 19 Jun 2012 17:01:02 -0400 Subject: Cleanup: move magic nubmers to new _httpinternal.h header file. --- indra/llcorehttp/CMakeLists.txt | 1 + indra/llcorehttp/_httpinternal.h | 82 ++++++++++++++++++++++++++++++++++ indra/llcorehttp/_httplibcurl.cpp | 17 ++++--- indra/llcorehttp/_httplibcurl.h | 14 ++---- indra/llcorehttp/_httpoperation.cpp | 3 +- indra/llcorehttp/_httpoprequest.cpp | 62 ++++++++++++------------- indra/llcorehttp/_httpoprequest.h | 6 +++ indra/llcorehttp/_httppolicy.cpp | 4 +- indra/llcorehttp/_httppolicy.h | 3 +- indra/llcorehttp/_httppolicyglobal.cpp | 10 +++-- indra/llcorehttp/_httpservice.cpp | 9 ++-- indra/llcorehttp/_refcounted.cpp | 4 +- indra/llcorehttp/bufferarray.cpp | 4 +- indra/llcorehttp/httpcommon.cpp | 2 +- indra/llcorehttp/httpoptions.cpp | 8 ++-- indra/llcorehttp/httprequest.cpp | 1 + indra/llcorehttp/httprequest.h | 6 +-- indra/newview/lltextureview.cpp | 2 +- 18 files changed, 161 insertions(+), 77 deletions(-) create mode 100644 indra/llcorehttp/_httpinternal.h (limited to 'indra/llcorehttp/httpcommon.cpp') diff --git a/indra/llcorehttp/CMakeLists.txt b/indra/llcorehttp/CMakeLists.txt index 9d8bae973e..acf4c0d6a3 100644 --- a/indra/llcorehttp/CMakeLists.txt +++ b/indra/llcorehttp/CMakeLists.txt @@ -53,6 +53,7 @@ set(llcorehttp_HEADER_FILES httpoptions.h httprequest.h httpresponse.h + _httpinternal.h _httplibcurl.h _httpopcancel.h _httpoperation.h diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h new file mode 100644 index 0000000000..bc0bd6a2ab --- /dev/null +++ b/indra/llcorehttp/_httpinternal.h @@ -0,0 +1,82 @@ +/** + * @file httpinternal.h + * @brief Implementation constants and magic numbers + * + * $LicenseInfo:firstyear=2012&license=viewerlgpl$ + * Second Life Viewer Source Code + * Copyright (C) 2012, Linden Research, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; + * version 2.1 of the License only. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + * + * Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA + * $/LicenseInfo$ + */ + +#ifndef _LLCORE_HTTP_INTERNAL_H_ +#define _LLCORE_HTTP_INTERNAL_H_ + + +// If you find this included in a public interface header, +// something wrong is probably happening. + + +namespace LLCore +{ + +// Maxium number of policy classes that can be defined. +// *FIXME: Currently limited to the default class, extend. +const int POLICY_CLASS_LIMIT = 1; + +// Debug/informational tracing. Used both +// as a global option and in per-request traces. +const int TRACE_OFF = 0; +const int TRACE_LOW = 1; +const int TRACE_CURL_HEADERS = 2; +const int TRACE_CURL_BODIES = 3; + +const int TRACE_MIN = TRACE_OFF; +const int TRACE_MAX = TRACE_CURL_BODIES; + +// Request retry limits +const int DEFAULT_RETRY_COUNT = 5; +const int LIMIT_RETRY_MIN = 0; +const int LIMIT_RETRY_MAX = 100; + +const int DEFAULT_HTTP_REDIRECTS = 10; + +// Timeout value used for both connect and protocol exchange. +// Retries and time-on-queue are not included and aren't +// accounted for. +const long DEFAULT_TIMEOUT = 30L; +const long LIMIT_TIMEOUT_MIN = 0L; +const long LIMIT_TIMEOUT_MAX = 3600L; + +// Limits on connection counts +const int DEFAULT_CONNECTIONS = 8; +const int LIMIT_CONNECTIONS_MIN = 1; +const int LIMIT_CONNECTIONS_MAX = 256; + +// Tuning parameters + +// Time worker thread sleeps after a pass through the +// request, ready and active queues. +const int LOOP_SLEEP_NORMAL_MS = 2; + +// Block allocation size (a tuning parameter) is found +// in bufferarray.h. + +} // end namespace LLCore + +#endif // _LLCORE_HTTP_INTERNAL_H_ diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index a176dd5b2a..65eb642056 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -31,6 +31,8 @@ #include "_httpoprequest.h" #include "_httppolicy.h" +#include "llhttpstatuscodes.h" + namespace LLCore { @@ -39,7 +41,8 @@ namespace LLCore HttpLibcurl::HttpLibcurl(HttpService * service) : mService(service) { - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + // *FIXME: Use active policy class count later + for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mMultiHandles); ++policy_class) { mMultiHandles[policy_class] = 0; } @@ -61,7 +64,7 @@ HttpLibcurl::~HttpLibcurl() mActiveOps.erase(item); } - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mMultiHandles); ++policy_class) { if (mMultiHandles[policy_class]) { @@ -89,7 +92,7 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() HttpService::ELoopSpeed ret(HttpService::REQUEST_SLEEP); // Give libcurl some cycles to do I/O & callbacks - for (int policy_class(0); policy_class < HttpRequest::POLICY_CLASS_LIMIT; ++policy_class) + for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mMultiHandles); ++policy_class) { if (! mMultiHandles[policy_class]) continue; @@ -144,7 +147,7 @@ HttpService::ELoopSpeed HttpLibcurl::processTransport() void HttpLibcurl::addOp(HttpOpRequest * op) { - llassert_always(op->mReqPolicy < HttpRequest::POLICY_CLASS_LIMIT); + llassert_always(op->mReqPolicy < POLICY_CLASS_LIMIT); llassert_always(mMultiHandles[op->mReqPolicy] != NULL); // Create standard handle @@ -159,7 +162,7 @@ void HttpLibcurl::addOp(HttpOpRequest * op) curl_multi_add_handle(mMultiHandles[op->mReqPolicy], op->mCurlHandle); op->mCurlActive = true; - if (op->mTracing > 0) + if (op->mTracing > TRACE_OFF) { HttpPolicy & policy(mService->getPolicy()); @@ -208,7 +211,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode } if (op->mStatus) { - int http_status(200); + int http_status(HTTP_OK); curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &http_status); op->mStatus = LLCore::HttpStatus(http_status); @@ -220,7 +223,7 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode op->mCurlHandle = NULL; // Tracing - if (op->mTracing > 0) + if (op->mTracing > TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, RequestComplete, Handle: " << static_cast(op) diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 16b68bde43..0d0c4cad6d 100644 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -36,6 +36,7 @@ #include "httprequest.h" #include "_httpservice.h" +#include "_httpinternal.h" namespace LLCore @@ -91,17 +92,8 @@ protected: protected: HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; - CURLM * mMultiHandles[HttpRequest::POLICY_CLASS_LIMIT]; -}; // end class HttpLibcurl - - -// --------------------------------------- -// Free functions -// --------------------------------------- - - -curl_slist * append_headers_to_slist(const HttpHeaders *, curl_slist * slist); - + CURLM * mMultiHandles[POLICY_CLASS_LIMIT]; +}; // end class HttpLibcurl } // end namespace LLCore diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 5a31bf90e7..0d9553434e 100644 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -33,6 +33,7 @@ #include "_httprequestqueue.h" #include "_httpreplyqueue.h" #include "_httpservice.h" +#include "_httpinternal.h" #include "lltimer.h" @@ -142,7 +143,7 @@ HttpStatus HttpOperation::cancel() void HttpOperation::addAsReply() { - if (mTracing > 0) + if (mTracing > TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, ToReplyQueue, Handle: " << static_cast(this) diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index ce41ebcce0..04a8e0baff 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -42,6 +42,7 @@ #include "_httppolicy.h" #include "_httppolicyglobal.h" #include "_httplibcurl.h" +#include "_httpinternal.h" #include "llhttpstatuscodes.h" #include "llproxy.h" @@ -73,13 +74,14 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line); -#if defined(WIN32) +#if LL_WINDOWS // Not available on windows where the legacy strtok interface // is thread-safe. char *strtok_r(char *str, const char *delim, char **saveptr); -#endif +#endif // LL_WINDOWS + } @@ -108,7 +110,7 @@ HttpOpRequest::HttpOpRequest() mReplyHeaders(NULL), mPolicyRetries(0), mPolicyRetryAt(HttpTime(0)), - mPolicyRetryLimit(5) + mPolicyRetryLimit(DEFAULT_RETRY_COUNT) { // *NOTE: As members are added, retry initialization/cleanup // may need to be extended in @prepareRequest(). @@ -334,8 +336,8 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, mProcFlags |= PF_SAVE_HEADERS; } mPolicyRetryLimit = options->getRetries(); - mPolicyRetryLimit = llclamp(mPolicyRetryLimit, 0, 100); - mTracing = (std::max)(mTracing, llclamp(options->getTrace(), 0, 3)); + mPolicyRetryLimit = llclamp(mPolicyRetryLimit, LIMIT_RETRY_MIN, LIMIT_RETRY_MAX); + mTracing = (std::max)(mTracing, llclamp(options->getTrace(), TRACE_MIN, TRACE_MAX)); } } @@ -384,7 +386,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_DNS_CACHE_TIMEOUT, 0); curl_easy_setopt(mCurlHandle, CURLOPT_AUTOREFERER, 1); curl_easy_setopt(mCurlHandle, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, 10); // *FIXME: parameterize this later + curl_easy_setopt(mCurlHandle, CURLOPT_MAXREDIRS, DEFAULT_HTTP_REDIRECTS); // *FIXME: parameterize this later curl_easy_setopt(mCurlHandle, CURLOPT_WRITEFUNCTION, writeCallback); curl_easy_setopt(mCurlHandle, CURLOPT_WRITEDATA, mCurlHandle); curl_easy_setopt(mCurlHandle, CURLOPT_READFUNCTION, readCallback); @@ -452,8 +454,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_INFILESIZE, data_size); curl_easy_setopt(mCurlHandle, CURLOPT_POSTFIELDS, (void *) NULL); mCurlHeaders = curl_slist_append(mCurlHeaders, "Expect:"); - mCurlHeaders = curl_slist_append(mCurlHeaders, "Connection: keep-alive"); - mCurlHeaders = curl_slist_append(mCurlHeaders, "Keep-alive: 300"); } break; @@ -463,7 +463,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } // Tracing - if (mTracing > 1) + if (mTracing >= TRACE_CURL_HEADERS) { curl_easy_setopt(mCurlHandle, CURLOPT_VERBOSE, 1); curl_easy_setopt(mCurlHandle, CURLOPT_DEBUGDATA, mCurlHandle); @@ -478,7 +478,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) char range_line[64]; -#if defined(WIN32) +#if LL_WINDOWS _snprintf_s(range_line, sizeof(range_line), sizeof(range_line) - 1, (mReqLength ? fmt1 : fmt2), (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1)); @@ -486,7 +486,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) snprintf(range_line, sizeof(range_line), (mReqLength ? fmt1 : fmt2), (unsigned long) mReqOffset, (unsigned long) (mReqOffset + mReqLength - 1)); -#endif // defined(WIN32) +#endif // LL_WINDOWS range_line[sizeof(range_line) - 1] = '\0'; mCurlHeaders = curl_slist_append(mCurlHeaders, range_line); } @@ -494,11 +494,11 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) mCurlHeaders = curl_slist_append(mCurlHeaders, "Pragma:"); // Request options - long timeout(30); + long timeout(DEFAULT_TIMEOUT); if (mReqOptions) { timeout = mReqOptions->getTimeout(); - timeout = llclamp(timeout, 0L, 3600L); + timeout = llclamp(timeout, LIMIT_TIMEOUT_MIN, LIMIT_TIMEOUT_MAX); } curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout); curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); @@ -599,11 +599,11 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi memcpy(hdr_buffer, hdr_data, frag_size); hdr_buffer[frag_size] = '\0'; -#if defined(WIN32) +#if LL_WINDOWS if (! _strnicmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len))) #else if (! strncasecmp(hdr_buffer, con_ran_line, (std::min)(frag_size, con_ran_line_len))) -#endif +#endif // LL_WINDOWS { unsigned int first(0), last(0), length(0); int status; @@ -654,7 +654,7 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe switch (info) { case CURLINFO_TEXT: - if (op->mTracing > 1) + if (op->mTracing >= TRACE_CURL_HEADERS) { tag = "TEXT"; escape_libcurl_debug_data(buffer, len, true, safe_line); @@ -663,7 +663,7 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe break; case CURLINFO_HEADER_IN: - if (op->mTracing > 1) + if (op->mTracing >= TRACE_CURL_HEADERS) { tag = "HEADERIN"; escape_libcurl_debug_data(buffer, len, true, safe_line); @@ -672,20 +672,20 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe break; case CURLINFO_HEADER_OUT: - if (op->mTracing > 1) + if (op->mTracing >= TRACE_CURL_HEADERS) { tag = "HEADEROUT"; - escape_libcurl_debug_data(buffer, len, true, safe_line); + escape_libcurl_debug_data(buffer, 2 * len, true, safe_line); // Goes out as one line logit = true; } break; case CURLINFO_DATA_IN: - if (op->mTracing > 1) + if (op->mTracing >= TRACE_CURL_HEADERS) { tag = "DATAIN"; logit = true; - if (op->mTracing > 2) + if (op->mTracing >= TRACE_CURL_BODIES) { escape_libcurl_debug_data(buffer, len, false, safe_line); } @@ -699,11 +699,11 @@ int HttpOpRequest::debugCallback(CURL * handle, curl_infotype info, char * buffe break; case CURLINFO_DATA_OUT: - if (op->mTracing > 1) + if (op->mTracing >= TRACE_CURL_HEADERS) { tag = "DATAOUT"; logit = true; - if (op->mTracing > 2) + if (op->mTracing >= TRACE_CURL_BODIES) { escape_libcurl_debug_data(buffer, len, false, safe_line); } @@ -755,22 +755,22 @@ int parse_content_range_header(char * buffer, if (! strtok_r(buffer, ": \t", &tok_state)) match = false; if (match && (tok = strtok_r(NULL, " \t", &tok_state))) -#if defined(WIN32) +#if LL_WINDOWS match = 0 == _stricmp("bytes", tok); #else match = 0 == strcasecmp("bytes", tok); -#endif +#endif // LL_WINDOWS if (match && ! (tok = strtok_r(NULL, " \t", &tok_state))) match = false; if (match) { unsigned int lcl_first(0), lcl_last(0), lcl_len(0); -#if defined(WIN32) +#if LL_WINDOWS if (3 == sscanf_s(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len)) #else if (3 == sscanf(tok, "%u-%u/%u", &lcl_first, &lcl_last, &lcl_len)) -#endif +#endif // LL_WINDOWS { if (lcl_first > lcl_last || lcl_last >= lcl_len) return -1; @@ -779,11 +779,11 @@ int parse_content_range_header(char * buffer, *length = lcl_len; return 0; } -#if defined(WIN32) +#if LL_WINDOWS if (2 == sscanf_s(tok, "%u-%u/*", &lcl_first, &lcl_last)) #else if (2 == sscanf(tok, "%u-%u/*", &lcl_first, &lcl_last)) -#endif +#endif // LL_WINDOWS { if (lcl_first > lcl_last) return -1; @@ -798,14 +798,14 @@ int parse_content_range_header(char * buffer, return 1; } -#if defined(WIN32) +#if LL_WINDOWS char *strtok_r(char *str, const char *delim, char ** savestate) { return strtok_s(str, delim, savestate); } -#endif +#endif // LL_WINDOWS void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 9278445763..f2b709a3a2 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -172,6 +172,12 @@ public: }; // end class HttpOpRequestCompare +// --------------------------------------- +// Free functions +// --------------------------------------- + +curl_slist * append_headers_to_slist(const HttpHeaders *, curl_slist * slist); + } // end namespace LLCore #endif // _LLCORE_HTTP_OPREQUEST_H_ diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 0e08d88276..4be9f1d45f 100644 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -126,7 +126,7 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() for (int policy_class(0); policy_class < LL_ARRAY_SIZE(mState); ++policy_class) { int active(transport.getActiveCountInClass(policy_class)); - int needed(8 - active); + int needed(DEFAULT_CONNECTIONS - active); // *FIXME: move to policy class HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); @@ -242,7 +242,7 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) { - if (policy_class < HttpRequest::POLICY_CLASS_LIMIT) + if (policy_class < POLICY_CLASS_LIMIT) // *FIXME: use actual active class count { return (mState[policy_class].mReadyQueue.size() + mState[policy_class].mRetryQueue.size()); diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 4114f64848..05de9303b5 100644 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -33,6 +33,7 @@ #include "_httpreadyqueue.h" #include "_httpretryqueue.h" #include "_httppolicyglobal.h" +#include "_httpinternal.h" namespace LLCore @@ -108,7 +109,7 @@ protected: HttpRetryQueue mRetryQueue; }; - State mState[HttpRequest::POLICY_CLASS_LIMIT]; + State mState[POLICY_CLASS_LIMIT]; HttpService * mService; // Naked pointer, not refcounted, not owner HttpPolicyGlobal mGlobalOptions; diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp index 6b1de38fd6..ca04839eaf 100644 --- a/indra/llcorehttp/_httppolicyglobal.cpp +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -26,6 +26,8 @@ #include "_httppolicyglobal.h" +#include "_httpinternal.h" + namespace LLCore { @@ -33,8 +35,8 @@ namespace LLCore HttpPolicyGlobal::HttpPolicyGlobal() : mSetMask(0UL), - mConnectionLimit(32L), - mTrace(0), + mConnectionLimit(DEFAULT_CONNECTIONS), + mTrace(TRACE_OFF), mUseLLProxy(0) {} @@ -64,11 +66,11 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) switch (opt) { case HttpRequest::GP_CONNECTION_LIMIT: - mConnectionLimit = value; + mConnectionLimit = llclamp(value, long(LIMIT_CONNECTIONS_MIN), long(LIMIT_CONNECTIONS_MAX)); break; case HttpRequest::GP_TRACE: - mTrace = llclamp(value, 0L, 3L); + mTrace = llclamp(value, long(TRACE_MIN), long(TRACE_MAX)); break; case HttpRequest::GP_LLPROXY: diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 87a78820f5..25f64acc42 100644 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -34,15 +34,12 @@ #include "_httppolicy.h" #include "_httplibcurl.h" #include "_thread.h" +#include "_httpinternal.h" #include "lltimer.h" #include "llthread.h" -// Tuning parameters -static const int LOOP_SLEEP_NORMAL_MS = 2; // Normal per-loop sleep in milliseconds - - namespace LLCore { @@ -230,11 +227,11 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) if (! mExitRequested) { // Setup for subsequent tracing - long tracing(0); + long tracing(TRACE_OFF); mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing); op->mTracing = (std::max)(op->mTracing, int(tracing)); - if (op->mTracing > 0) + if (op->mTracing > TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, FromRequestQueue, Handle: " << static_cast(op) diff --git a/indra/llcorehttp/_refcounted.cpp b/indra/llcorehttp/_refcounted.cpp index 11d75fdf97..e7d0b72741 100644 --- a/indra/llcorehttp/_refcounted.cpp +++ b/indra/llcorehttp/_refcounted.cpp @@ -30,11 +30,11 @@ namespace LLCoreInt { -#if ! defined(WIN32) +#if ! LL_WINDOWS const S32 RefCounted::NOT_REF_COUNTED; -#endif // ! defined(WIN32) +#endif // ! LL_WINDOWS RefCounted::~RefCounted() {} diff --git a/indra/llcorehttp/bufferarray.cpp b/indra/llcorehttp/bufferarray.cpp index ae92057df0..5eaa60c9ba 100644 --- a/indra/llcorehttp/bufferarray.cpp +++ b/indra/llcorehttp/bufferarray.cpp @@ -85,9 +85,9 @@ public: // ================================== -#if ! defined(WIN32) +#if ! LL_WINDOWS const size_t BufferArray::BLOCK_ALLOC_SIZE; -#endif // ! defined(WIN32) +#endif // ! LL_WINDOWS BufferArray::BufferArray() : LLCoreInt::RefCounted(true), diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 9f17b5c842..1b18976359 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -54,9 +54,9 @@ std::string HttpStatus::toHex() const result.fill('0'); result << std::hex << operator unsigned long(); return result.str(); - } + std::string HttpStatus::toString() const { static const char * llcore_errors[] = diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index c11d89e619..f2771c1f29 100644 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -26,6 +26,8 @@ #include "httpoptions.h" +#include "_httpinternal.h" + namespace LLCore { @@ -34,9 +36,9 @@ namespace LLCore HttpOptions::HttpOptions() : RefCounted(true), mWantHeaders(false), - mTracing(0), - mTimeout(30), - mRetries(5) + mTracing(TRACE_OFF), + mTimeout(DEFAULT_TIMEOUT), + mRetries(DEFAULT_RETRY_COUNT) {} diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 2036ecfd1c..e906ff8a1e 100644 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -108,6 +108,7 @@ HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::stri HttpRequest::policy_t HttpRequest::createPolicyClass() { + // *FIXME: Implement classes policy_t policy_id = 1; return policy_id; diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 4e78ed3719..24fff24b83 100644 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -102,10 +102,6 @@ public: /// eventual service for any HTTP request. static const int DEFAULT_POLICY_ID = 0; - /// Maximum number of policies that may be defined. No policy - /// ID will equal or exceed this value. - static const int POLICY_CLASS_LIMIT = 1; - enum EGlobalPolicy { /// Maximum number of connections the library will use to @@ -163,7 +159,7 @@ public: /// Create a new policy class into which requests can be made. /// /// @return If positive, the policy_id used to reference - /// the class in other methods. If -1, an error + /// the class in other methods. If 0, an error /// occurred and @see getStatus() may provide more /// detail on the reason. static policy_t createPolicyClass(); diff --git a/indra/newview/lltextureview.cpp b/indra/newview/lltextureview.cpp index a4227ec2ff..ae2da84edd 100644 --- a/indra/newview/lltextureview.cpp +++ b/indra/newview/lltextureview.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2001&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2010, Linden Research, Inc. + * Copyright (C) 2012, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public -- cgit v1.3 From d238341afaecedfe227141126c4c35dcde4a0671 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 16 Jul 2012 11:53:04 -0400 Subject: SH-3189 Remove/improve naive data structures When releasing HTTP waiters, avoid unnecessary sort activity. For Content-Type in responses, let libcurl do the work and removed my parsing of headers. Drop Content-Encoding as libcurl will deal with that. If anyone is interested, they can parse. --- indra/llcorehttp/_httplibcurl.cpp | 18 ++++++++++- indra/llcorehttp/_httpoprequest.cpp | 48 ++--------------------------- indra/llcorehttp/_httpoprequest.h | 2 -- indra/llcorehttp/httpcommon.cpp | 3 +- indra/llcorehttp/httpcommon.h | 5 ++- indra/llcorehttp/httpresponse.h | 9 ++---- indra/llcorehttp/tests/test_httprequest.hpp | 3 +- indra/newview/lltexturefetch.cpp | 22 ++++++++++--- 8 files changed, 48 insertions(+), 62 deletions(-) (limited to 'indra/llcorehttp/httpcommon.cpp') diff --git a/indra/llcorehttp/_httplibcurl.cpp b/indra/llcorehttp/_httplibcurl.cpp index 3c69ae1c96..e031efbc91 100644 --- a/indra/llcorehttp/_httplibcurl.cpp +++ b/indra/llcorehttp/_httplibcurl.cpp @@ -280,7 +280,23 @@ bool HttpLibcurl::completeRequest(CURLM * multi_handle, CURL * handle, CURLcode int http_status(HTTP_OK); curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &http_status); - op->mStatus = LLCore::HttpStatus(http_status); + if (http_status >= 100 && http_status <= 999) + { + char * cont_type(NULL); + curl_easy_getinfo(handle, CURLINFO_CONTENT_TYPE, &cont_type); + if (cont_type) + { + op->mReplyConType = cont_type; + } + op->mStatus = HttpStatus(http_status); + } + else + { + LL_WARNS("CoreHttp") << "Invalid HTTP response code (" + << http_status << ") received from server." + << LL_ENDL; + op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INVALID_HTTP_STATUS); + } } // Detach from multi and recycle handle diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 1854d7ada4..1a770f67be 100644 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -228,7 +228,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) // Got an explicit offset/length in response response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); } - response->setContent(mReplyConType, mReplyConEncode); + response->setContentType(mReplyConType); mUserHandler->onCompleted(static_cast(this), response); @@ -316,7 +316,7 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, HttpOptions * options, HttpHeaders * headers) { - mProcFlags = PF_SCAN_CONTENT_HEADERS; // Always scan for content headers + mProcFlags = 0U; mReqPolicy = policy_id; mReqPriority = priority; mReqURL = url; @@ -379,7 +379,6 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) mReplyHeaders = NULL; } mReplyConType.clear(); - mReplyConEncode.clear(); // *FIXME: better error handling later HttpStatus status; @@ -542,7 +541,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); - if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_SCAN_CONTENT_HEADERS)) + if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) { curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); @@ -620,8 +619,6 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi op->mReplyOffset = 0; op->mReplyLength = 0; op->mReplyFullLength = 0; - op->mReplyConType.clear(); - op->mReplyConEncode.clear(); op->mStatus = HttpStatus(); if (op->mReplyHeaders) { @@ -688,45 +685,6 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi } } - // Detect and parse 'Content-Type' and 'Content-Encoding' headers - if (op->mProcFlags & PF_SCAN_CONTENT_HEADERS) - { - if (wanted_hdr_size > con_type_line_len && - ! os_strncasecmp(hdr_data, con_type_line, con_type_line_len)) - { - // Found 'Content-Type:', extract single-token value - std::string rhs(hdr_data + con_type_line_len, wanted_hdr_size - con_type_line_len); - std::string::size_type begin(0), end(rhs.size()), pos; - - if ((pos = rhs.find_first_not_of(hdr_whitespace)) != std::string::npos) - { - begin = pos; - } - if ((pos = rhs.find_first_of(hdr_whitespace, begin)) != std::string::npos) - { - end = pos; - } - op->mReplyConType.assign(rhs, begin, end - begin); - } - else if (wanted_hdr_size > con_enc_line_len && - ! os_strncasecmp(hdr_data, con_enc_line, con_enc_line_len)) - { - // Found 'Content-Encoding:', extract single-token value - std::string rhs(hdr_data + con_enc_line_len, wanted_hdr_size - con_enc_line_len); - std::string::size_type begin(0), end(rhs.size()), pos; - - if ((pos = rhs.find_first_not_of(hdr_whitespace)) != std::string::npos) - { - begin = pos; - } - if ((pos = rhs.find_first_of(hdr_whitespace, begin)) != std::string::npos) - { - end = pos; - } - op->mReplyConEncode.assign(rhs, begin, end - begin); - } - } - return hdr_size; } diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 200b925c4e..36dc5dc876 100644 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -138,7 +138,6 @@ protected: unsigned int mProcFlags; static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; static const unsigned int PF_SAVE_HEADERS = 0x00000002U; - static const unsigned int PF_SCAN_CONTENT_HEADERS = 0x00000004U; public: // Request data @@ -165,7 +164,6 @@ public: size_t mReplyFullLength; HttpHeaders * mReplyHeaders; std::string mReplyConType; - std::string mReplyConEncode; // Policy data int mPolicyRetries; diff --git a/indra/llcorehttp/httpcommon.cpp b/indra/llcorehttp/httpcommon.cpp index 1b18976359..f2fcbf77a3 100644 --- a/indra/llcorehttp/httpcommon.cpp +++ b/indra/llcorehttp/httpcommon.cpp @@ -69,7 +69,8 @@ std::string HttpStatus::toString() const "Request handle not found", "Invalid datatype for argument or option", "Option has not been explicitly set", - "Option is not dynamic and must be set early" + "Option is not dynamic and must be set early", + "Invalid HTTP status code received from server" }; static const int llcore_errors_count(sizeof(llcore_errors) / sizeof(llcore_errors[0])); diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index 576a113e54..dd5798edf9 100644 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -149,7 +149,10 @@ enum HttpError HE_OPT_NOT_SET = 7, // Option not dynamic, must be set during init phase - HE_OPT_NOT_DYNAMIC = 8 + HE_OPT_NOT_DYNAMIC = 8, + + // Invalid HTTP status code returned by server + HE_INVALID_HTTP_STATUS = 9 }; // end enum HttpError diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index 3d35050c17..4a481db6ac 100644 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -134,16 +134,14 @@ public: } /// - void getContent(std::string & con_type, std::string & con_encode) const + const std::string & getContentType() const { - con_type = mContentType; - con_encode = mContentEncoding; + return mContentType; } - void setContent(const std::string & con_type, const std::string & con_encode) + void setContentType(const std::string & con_type) { mContentType = con_type; - mContentEncoding = con_encode; } protected: @@ -155,7 +153,6 @@ protected: BufferArray * mBufferArray; HttpHeaders * mHeaders; std::string mContentType; - std::string mContentEncoding; }; diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 0f9eb93996..1acf4f9d4b 100644 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -147,8 +147,7 @@ public: if (! mCheckContentType.empty()) { ensure("Response required with content type check", response != NULL); - std::string con_type, con_enc; - response->getContent(con_type, con_enc); + std::string con_type(response->getContentType()); ensure("Content-Type as expected (" + mCheckContentType + ")", mCheckContentType == con_type); } diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index 8314031f14..225ea46558 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -3342,6 +3342,17 @@ void LLTextureFetch::removeHttpWaiter(const LLUUID & tid) mNetworkQueueMutex.unlock(); // -Mfnq } +// Release as many requests as permitted from the WAIT_HTTP_RESOURCE2 +// state to the SEND_HTTP_REQ state based on their current priority. +// +// This data structures and code associated with this looks a bit +// indirect and naive but it's done in the name of safety. An +// ordered container may become invalid from time to time due to +// priority changes caused by actions in other threads. State itself +// could also suffer the same fate with canceled operations. Even +// done this way, I'm not fully trusting we're truly safe. This +// module is due for a major refactoring and we'll deal with it then. +// // Threads: Ttf // Locks: -Mw (must not hold any worker when called) void LLTextureFetch::releaseHttpWaiters() @@ -3384,12 +3395,15 @@ void LLTextureFetch::releaseHttpWaiters() tids2.push_back(worker); } } - - // Sort into priority order - LLTextureFetchWorker::Compare compare; - std::sort(tids2.begin(), tids2.end(), compare); tids.clear(); + // Sort into priority order, if necessary and only as much as needed + if (tids2.size() > mHttpSemaphore) + { + LLTextureFetchWorker::Compare compare; + std::partial_sort(tids2.begin(), tids2.begin() + mHttpSemaphore, tids2.end(), compare); + } + // Release workers up to the high water mark. Since we aren't // holding any locks at this point, we can be in competition // with other callers. Do defensive things like getting -- cgit v1.3