From 2df0a2494691ebfdd305e6a5ee280dd758a1b337 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 7 May 2013 16:18:31 +0000 Subject: SH-4139 Convert http downloaders and responders to llcorehttp patterns Initial work completed on linux, moving over to windows to do debug and refinement. This includes 5/6 handlers based on existing responders and use of llcorehttp for the mesh header fetch. --- indra/newview/llappcorehttp.cpp | 48 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) (limited to 'indra/newview/llappcorehttp.cpp') diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 0d7d41304d..4386e3283b 100644 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -38,7 +38,9 @@ LLAppCoreHttp::LLAppCoreHttp() mStopHandle(LLCORE_HTTP_HANDLE_INVALID), mStopRequested(0.0), mStopped(false), - mPolicyDefault(-1) + mPolicyDefault(-1), + mPolicyTexture(-1), + mPolicyMesh(-1) {} @@ -95,6 +97,9 @@ void LLAppCoreHttp::init() // Setup default policy and constrain if directed to mPolicyDefault = LLCore::HttpRequest::DEFAULT_POLICY_ID; + + // Texture policy will use default for now. + mPolicyTexture = mPolicyDefault; static const std::string texture_concur("TextureFetchConcurrency"); if (gSavedSettings.controlExists(texture_concur)) { @@ -103,7 +108,7 @@ void LLAppCoreHttp::init() if (concur > 0) { LLCore::HttpStatus status; - status = LLCore::HttpRequest::setPolicyClassOption(mPolicyDefault, + status = LLCore::HttpRequest::setPolicyClassOption(mPolicyTexture, LLCore::HttpRequest::CP_CONNECTION_LIMIT, concur); if (! status) @@ -120,6 +125,43 @@ void LLAppCoreHttp::init() } } } + + // Create the mesh class + mPolicyMesh = LLCore::HttpRequest::createPolicyClass(); + if (! mPolicyMesh) + { + LL_WARNS("Init") << "Failed to create HTTP policy class for Mesh. Using default policy." + << LL_ENDL; + mPolicyMesh = mPolicyDefault; + } + else + { + static const std::string mesh_concur("MeshMaxConcurrentRequests"); + if (gSavedSettings.controlExists(mesh_concur)) + { + U32 setting(llmin(gSavedSettings.getU32(mesh_concur), U32(32))); + + if (setting > 0) + { + LLCore::HttpStatus status; + status = LLCore::HttpRequest::setPolicyClassOption(mPolicyMesh, + LLCore::HttpRequest::CP_CONNECTION_LIMIT, + setting); + if (! status) + { + LL_WARNS("Init") << "Unable to set mesh fetch concurrency. Reason: " + << status.toString() + << LL_ENDL; + } + else + { + LL_INFOS("Init") << "Application settings overriding default mesh fetch concurrency. New value: " + << setting + << LL_ENDL; + } + } + } + } // Kick the thread status = LLCore::HttpRequest::startThread(); -- cgit v1.3 From f55f4bf1b25d14bcd3d956e6c170ece3880ad5f5 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 8 May 2013 18:37:08 -0400 Subject: SH-4163 Run an initial series of 'B' tests on the mesh downloader code Bleh, had some old initialization code in place that meant I was using 32 connections. (Always verify with 'netstat'...) Logic is now to use 1/4 of MeshMaxConncurrentRequests to head in the direction of 8 at a time. Full count is used to implement a high-water level keeping llcorehttp in work. --- indra/newview/llappcorehttp.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'indra/newview/llappcorehttp.cpp') diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 4386e3283b..142344e277 100644 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -139,7 +139,8 @@ void LLAppCoreHttp::init() static const std::string mesh_concur("MeshMaxConcurrentRequests"); if (gSavedSettings.controlExists(mesh_concur)) { - U32 setting(llmin(gSavedSettings.getU32(mesh_concur), U32(32))); + U32 setting(llmin(gSavedSettings.getU32(mesh_concur), 256U) / 4U); + setting = llmax(setting, 2U); if (setting > 0) { -- cgit v1.3 From 626752beab7c12a355ab707d70aba6f4fe096c10 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 19 Jun 2013 13:55:54 -0400 Subject: SH-4252 Add second policy class for large mesh asset downloads Added second mesh class as well as an asset upload class. Refactored initialization to use less code and more data to cleanly get http started. Modified mesh to use the new http class for large requests (>2MB for now). Added additional timeout setting to llcorehttp to distinguish connection timeout from transport timeout and are now using transport timeout values for large asset downloads that may need more time. --- indra/llcorehttp/_httpinternal.h | 3 +- indra/llcorehttp/_httpoprequest.cpp | 15 ++- indra/llcorehttp/_httpoprequest.h | 3 +- indra/llcorehttp/_httppolicy.cpp | 6 ++ indra/llcorehttp/httpoptions.cpp | 9 +- indra/llcorehttp/httpoptions.h | 9 +- indra/llcorehttp/httpresponse.cpp | 6 +- indra/llcorehttp/httpresponse.h | 17 +++- indra/newview/llappcorehttp.cpp | 156 ++++++++++++++++------------ indra/newview/llappcorehttp.h | 36 +++---- indra/newview/llmeshrepository.cpp | 197 +++++++++++++++++++++--------------- indra/newview/llmeshrepository.h | 21 +++- indra/newview/lltexturefetch.cpp | 4 +- 13 files changed, 301 insertions(+), 181 deletions(-) (limited to 'indra/newview/llappcorehttp.cpp') diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index 30b0905c12..d60996756f 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -98,7 +98,7 @@ namespace LLCore // Maxium number of policy classes that can be defined. // *TODO: Currently limited to the default class + 1, extend. -const int HTTP_POLICY_CLASS_LIMIT = 2; +const int HTTP_POLICY_CLASS_LIMIT = 4; // Debug/informational tracing. Used both // as a global option and in per-request traces. @@ -129,6 +129,7 @@ const int HTTP_REDIRECTS_DEFAULT = 10; // Retries and time-on-queue are not included and aren't // accounted for. const long HTTP_REQUEST_TIMEOUT_DEFAULT = 30L; +const long HTTP_REQUEST_XFER_TIMEOUT_DEFAULT = 0L; const long HTTP_REQUEST_TIMEOUT_MIN = 0L; const long HTTP_REQUEST_TIMEOUT_MAX = 3600L; diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index 51a8eaf998..d403b2d249 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -110,6 +110,7 @@ HttpOpRequest::HttpOpRequest() mReplyFullLength(0), mReplyHeaders(NULL), mPolicyRetries(0), + mPolicy503Retries(0), mPolicyRetryAt(HttpTime(0)), mPolicyRetryLimit(HTTP_RETRY_COUNT_DEFAULT) { @@ -224,6 +225,7 @@ void HttpOpRequest::visitNotifier(HttpRequest * request) response->setRange(mReplyOffset, mReplyLength, mReplyFullLength); } response->setContentType(mReplyConType); + response->setRetries(mPolicyRetries, mPolicy503Retries); mUserHandler->onCompleted(static_cast(this), response); @@ -524,12 +526,19 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // Request options long timeout(HTTP_REQUEST_TIMEOUT_DEFAULT); + long xfer_timeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT); if (mReqOptions) - { + { timeout = mReqOptions->getTimeout(); timeout = llclamp(timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); + xfer_timeout = mReqOptions->getTransferTimeout(); + xfer_timeout = llclamp(xfer_timeout, HTTP_REQUEST_TIMEOUT_MIN, HTTP_REQUEST_TIMEOUT_MAX); + } + if (xfer_timeout == 0L) + { + xfer_timeout = timeout; } - curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, timeout); + curl_easy_setopt(mCurlHandle, CURLOPT_TIMEOUT, xfer_timeout); curl_easy_setopt(mCurlHandle, CURLOPT_CONNECTTIMEOUT, timeout); // Request headers diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 7b65d17783..831e5bebf7 100755 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -187,6 +187,7 @@ public: // Policy data int mPolicyRetries; + int mPolicy503Retries; HttpTime mPolicyRetryAt; int mPolicyRetryLimit; }; // end class HttpOpRequest diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 76c1e22431..54c9c6bb1b 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -140,6 +140,7 @@ void HttpPolicy::addOp(HttpOpRequest * op) const int policy_class(op->mReqPolicy); op->mPolicyRetries = 0; + op->mPolicy503Retries = 0; mState[policy_class].mReadyQueue.push(op); } @@ -155,6 +156,7 @@ void HttpPolicy::retryOp(HttpOpRequest * op) 5000000 // ... to every 5.0 S. }; static const int delta_max(int(LL_ARRAY_SIZE(retry_deltas)) - 1); + static const HttpStatus error_503(503); const HttpTime now(totalTime()); const int policy_class(op->mReqPolicy); @@ -162,6 +164,10 @@ void HttpPolicy::retryOp(HttpOpRequest * op) const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); op->mPolicyRetryAt = now + delta; ++op->mPolicyRetries; + if (error_503 == op->mStatus) + { + ++op->mPolicy503Retries; + } LL_WARNS("CoreHttp") << "HTTP request " << static_cast(op) << " retry " << op->mPolicyRetries << " scheduled for +" << (delta / HttpTime(1000)) diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index 1699d19f8d..4dcd862ca4 100755 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -38,6 +38,7 @@ HttpOptions::HttpOptions() mWantHeaders(false), mTracing(HTTP_TRACE_OFF), mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), + mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT), mRetries(HTTP_RETRY_COUNT_DEFAULT) {} @@ -64,6 +65,12 @@ void HttpOptions::setTimeout(unsigned int timeout) } +void HttpOptions::setTransferTimeout(unsigned int timeout) +{ + mTransferTimeout = timeout; +} + + void HttpOptions::setRetries(unsigned int retries) { mRetries = retries; diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h index 97e46a8cd3..623d71d3e6 100755 --- a/indra/llcorehttp/httpoptions.h +++ b/indra/llcorehttp/httpoptions.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -86,6 +86,12 @@ public: return mTimeout; } + void setTransferTimeout(unsigned int timeout); + unsigned int getTransferTimeout() const + { + return mTransferTimeout; + } + void setRetries(unsigned int retries); unsigned int getRetries() const { @@ -96,6 +102,7 @@ protected: bool mWantHeaders; int mTracing; unsigned int mTimeout; + unsigned int mTransferTimeout; unsigned int mRetries; }; // end class HttpOptions diff --git a/indra/llcorehttp/httpresponse.cpp b/indra/llcorehttp/httpresponse.cpp index a552e48a1b..c974395b0a 100755 --- a/indra/llcorehttp/httpresponse.cpp +++ b/indra/llcorehttp/httpresponse.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -39,7 +39,9 @@ HttpResponse::HttpResponse() mReplyLength(0U), mReplyFullLength(0U), mBufferArray(NULL), - mHeaders(NULL) + mHeaders(NULL), + mRetries(0U), + m503Retries(0U) {} diff --git a/indra/llcorehttp/httpresponse.h b/indra/llcorehttp/httpresponse.h index 4a481db6ac..a7f296e03f 100755 --- a/indra/llcorehttp/httpresponse.h +++ b/indra/llcorehttp/httpresponse.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -144,6 +144,19 @@ public: mContentType = con_type; } + /// Get and set retry attempt information on the request. + void getRetries(unsigned int * retries, unsigned int * retries_503) const + { + *retries = mRetries; + *retries_503 = m503Retries; + } + + void setRetries(unsigned int retries, unsigned int retries_503) + { + mRetries = retries; + m503Retries = retries_503; + } + protected: // Response data here HttpStatus mStatus; @@ -153,6 +166,8 @@ protected: BufferArray * mBufferArray; HttpHeaders * mHeaders; std::string mContentType; + unsigned int mRetries; + unsigned int m503Retries; }; diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 142344e277..b601b31d21 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -37,11 +37,13 @@ LLAppCoreHttp::LLAppCoreHttp() : mRequest(NULL), mStopHandle(LLCORE_HTTP_HANDLE_INVALID), mStopRequested(0.0), - mStopped(false), - mPolicyDefault(-1), - mPolicyTexture(-1), - mPolicyMesh(-1) -{} + mStopped(false) +{ + for (int i(0); i < LL_ARRAY_SIZE(mPolicies); ++i) + { + mPolicies[i] = LLCore::HttpRequest::DEFAULT_POLICY_ID; + } +} LLAppCoreHttp::~LLAppCoreHttp() @@ -53,11 +55,43 @@ LLAppCoreHttp::~LLAppCoreHttp() void LLAppCoreHttp::init() { + static const struct + { + EAppPolicy mPolicy; + U32 mDefault; + U32 mMin; + U32 mMax; + U32 mDivisor; + std::string mKey; + const char * mUsage; + } init_data[] = // Default and dynamic values for classes + { + { + AP_TEXTURE, 8, 1, 12, 1, + "TextureFetchConcurrency", + "texture fetch" + }, + { + AP_MESH, 8, 1, 32, 4, + "MeshMaxConcurrentRequests", + "mesh fetch" + }, + { + AP_LARGE_MESH, 2, 1, 8, 1, + "", + "large mesh fetch" + }, + { + AP_UPLOADS, 2, 1, 8, 1, + "", + "asset upload" + } + }; + LLCore::HttpStatus status = LLCore::HttpRequest::createService(); if (! status) { - LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " - << status.toString() + LL_ERRS("Init") << "Failed to initialize HTTP services. Reason: " << status.toString() << LL_ENDL; } @@ -66,8 +100,7 @@ void LLAppCoreHttp::init() gDirUtilp->getCAFile()); if (! status) { - LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " - << status.toString() + LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " << status.toString() << LL_ENDL; } @@ -77,8 +110,7 @@ void LLAppCoreHttp::init() status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); if (! status) { - LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " - << status.toString() + LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " << status.toString() << LL_ENDL; } @@ -96,80 +128,72 @@ void LLAppCoreHttp::init() } // Setup default policy and constrain if directed to - mPolicyDefault = LLCore::HttpRequest::DEFAULT_POLICY_ID; + mPolicies[AP_DEFAULT] = LLCore::HttpRequest::DEFAULT_POLICY_ID; - // Texture policy will use default for now. - mPolicyTexture = mPolicyDefault; - static const std::string texture_concur("TextureFetchConcurrency"); - if (gSavedSettings.controlExists(texture_concur)) + // Setup additional policies based on table and some special rules + // *TODO: Make these configurations dynamic later + for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { - U32 concur(llmin(gSavedSettings.getU32(texture_concur), U32(12))); + const EAppPolicy policy(init_data[i].mPolicy); - if (concur > 0) + // Create a policy class but use default for texture for now. + // This also has the side-effect of initializing the default + // class to desired values. + if (AP_TEXTURE == policy) { - LLCore::HttpStatus status; - status = LLCore::HttpRequest::setPolicyClassOption(mPolicyTexture, - LLCore::HttpRequest::CP_CONNECTION_LIMIT, - concur); - if (! status) - { - LL_WARNS("Init") << "Unable to set texture fetch concurrency. Reason: " - << status.toString() - << LL_ENDL; - } - else + mPolicies[policy] = mPolicies[AP_DEFAULT]; + } + else + { + mPolicies[policy] = LLCore::HttpRequest::createPolicyClass(); + if (! mPolicies[policy]) { - LL_INFOS("Init") << "Application settings overriding default texture fetch concurrency. New value: " - << concur + // Use default policy (but don't accidentally modify default) + LL_WARNS("Init") << "Failed to create HTTP policy class for " << init_data[i].mUsage + << ". Using default policy." << LL_ENDL; + mPolicies[policy] = mPolicies[AP_DEFAULT]; + continue; } } - } - // Create the mesh class - mPolicyMesh = LLCore::HttpRequest::createPolicyClass(); - if (! mPolicyMesh) - { - LL_WARNS("Init") << "Failed to create HTTP policy class for Mesh. Using default policy." - << LL_ENDL; - mPolicyMesh = mPolicyDefault; - } - else - { - static const std::string mesh_concur("MeshMaxConcurrentRequests"); - if (gSavedSettings.controlExists(mesh_concur)) + // Get target connection concurrency value + U32 setting(init_data[i].mDefault); + if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) { - U32 setting(llmin(gSavedSettings.getU32(mesh_concur), 256U) / 4U); - setting = llmax(setting, 2U); - - if (setting > 0) + U32 new_setting(gSavedSettings.getU32(init_data[i].mKey)); + if (new_setting) { - LLCore::HttpStatus status; - status = LLCore::HttpRequest::setPolicyClassOption(mPolicyMesh, - LLCore::HttpRequest::CP_CONNECTION_LIMIT, - setting); - if (! status) - { - LL_WARNS("Init") << "Unable to set mesh fetch concurrency. Reason: " - << status.toString() - << LL_ENDL; - } - else - { - LL_INFOS("Init") << "Application settings overriding default mesh fetch concurrency. New value: " - << setting - << LL_ENDL; - } + // Treat zero settings as an ask for default + setting = new_setting / init_data[i].mDivisor; + setting = llclamp(setting, init_data[i].mMin, init_data[i].mMax); } } + + // Set it and report + LLCore::HttpStatus status; + status = LLCore::HttpRequest::setPolicyClassOption(mPolicies[policy], + LLCore::HttpRequest::CP_CONNECTION_LIMIT, + setting); + if (! status) + { + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " concurrency. Reason: " << status.toString() + << LL_ENDL; + } + else if (setting != init_data[i].mDefault) + { + LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + } } // Kick the thread status = LLCore::HttpRequest::startThread(); if (! status) { - LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " - << status.toString() + LL_ERRS("Init") << "Failed to start HTTP servicing thread. Reason: " << status.toString() << LL_ENDL; } diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index d90af9e5ca..532e1f5cb0 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -40,6 +40,19 @@ // as a singleton and static construction is fine. class LLAppCoreHttp : public LLCore::HttpHandler { +public: + typedef LLCore::HttpRequest::policy_t policy_t; + + enum EAppPolicy + { + AP_DEFAULT, + AP_TEXTURE, + AP_MESH, + AP_LARGE_MESH, + AP_UPLOADS, + AP_COUNT // Must be last + }; + public: LLAppCoreHttp(); ~LLAppCoreHttp(); @@ -65,22 +78,11 @@ public: // Notification when the stop request is complete. virtual void onCompleted(LLCore::HttpHandle handle, LLCore::HttpResponse * response); - // Retrieve the policy class for default operations. - int getPolicyDefault() const - { - return mPolicyDefault; - } - - // Get the texture fetch policy class. - int getPolicyTexture() const - { - return mPolicyTexture; - } - - // Get the mesh fetch policy class. - int getPolicyMesh() const + // Retrieve a policy class identifier for desired + // application function. + policy_t getPolicy(EAppPolicy policy) const { - return mPolicyMesh; + return mPolicies[policy]; } private: @@ -91,9 +93,7 @@ private: LLCore::HttpHandle mStopHandle; F64 mStopRequested; bool mStopped; - int mPolicyDefault; - int mPolicyTexture; - int mPolicyMesh; + policy_t mPolicies[AP_COUNT]; }; diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 702e940983..f0ec97a34d 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -80,6 +80,10 @@ LLMeshRepository gMeshRepo; const S32 MESH_HEADER_SIZE = 4096; const U32 MAX_MESH_REQUESTS_PER_SECOND = 100; +const S32 REQUEST_HIGH_WATER_MIN = 32; +const S32 REQUEST_LOW_WATER_MIN = 16; +const U32 LARGE_MESH_FETCH_THRESHOLD = 1U << 21; // Size at which requests goes to narrow/slow queue +const long LARGE_MESH_XFER_TIMEOUT = 240L; // Seconds to complete xfer // Maximum mesh version to support. Three least significant digits are reserved for the minor version, // with major version changes indicating a format change that is not backwards compatible and should not @@ -210,6 +214,8 @@ void get_vertex_buffer_from_mesh(LLCDMeshData& mesh, LLModel::PhysicsMesh& res, S32 LLMeshRepoThread::sActiveHeaderRequests = 0; S32 LLMeshRepoThread::sActiveLODRequests = 0; U32 LLMeshRepoThread::sMaxConcurrentRequests = 1; +S32 LLMeshRepoThread::sRequestLowWater = REQUEST_LOW_WATER_MIN; +S32 LLMeshRepoThread::sRequestHighWater = REQUEST_HIGH_WATER_MIN; class LLMeshHandlerBase : public LLCore::HttpHandler { @@ -548,25 +554,37 @@ public: LLMeshRepoThread::LLMeshRepoThread() : LLThread("mesh repo"), - mCurlRequest(NULL), mWaiting(false), mHttpRequest(NULL), mHttpOptions(NULL), + mHttpLargeOptions(NULL), mHttpHeaders(NULL), - mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID) + mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpLargePolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpPriority(0), + mHttpGetCount(0U), + mHttpLargeGetCount(0U) { mMutex = new LLMutex(NULL); mHeaderMutex = new LLMutex(NULL); mSignal = new LLCondition(NULL); mHttpRequest = new LLCore::HttpRequest; mHttpOptions = new LLCore::HttpOptions; + mHttpLargeOptions = new LLCore::HttpOptions; + mHttpLargeOptions->setTransferTimeout(LARGE_MESH_XFER_TIMEOUT); mHttpHeaders = new LLCore::HttpHeaders; mHttpHeaders->mHeaders.push_back("Accept: application/vnd.ll.mesh"); - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicyMesh(); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_LARGE_MESH); } + LLMeshRepoThread::~LLMeshRepoThread() { + LL_INFOS("Mesh") << "Small GETs issued: " + << mHttpGetCount << ", Large GETs issued: " + << mHttpLargeGetCount << LL_ENDL; + for (http_request_set::iterator iter(mHttpRequestSet.begin()); iter != mHttpRequestSet.end(); ++iter) @@ -584,6 +602,11 @@ LLMeshRepoThread::~LLMeshRepoThread() mHttpOptions->release(); mHttpOptions = NULL; } + if (mHttpLargeOptions) + { + mHttpLargeOptions->release(); + mHttpLargeOptions = NULL; + } delete mHttpRequest; mHttpRequest = NULL; delete mMutex; @@ -596,7 +619,6 @@ LLMeshRepoThread::~LLMeshRepoThread() void LLMeshRepoThread::run() { - mCurlRequest = new LLCurlRequest(); LLCDResult res = LLConvexDecomposition::initThread(); if (res != LLCD_OK) { @@ -627,7 +649,7 @@ void LLMeshRepoThread::run() // NOTE: throttling intentionally favors LOD requests over header requests - while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveLODRequests < sMaxConcurrentRequests) + while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND) { if (mMutex) { @@ -646,7 +668,7 @@ void LLMeshRepoThread::run() } } - while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && sActiveHeaderRequests < sMaxConcurrentRequests) + while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND) { if (mMutex) { @@ -701,8 +723,6 @@ void LLMeshRepoThread::run() } mPhysicsShapeRequests = incomplete; } - - mCurlRequest->process(); } } @@ -716,9 +736,6 @@ void LLMeshRepoThread::run() { llwarns << "convex decomposition unable to be quit" << llendl; } - - delete mCurlRequest; - mCurlRequest = NULL; } void LLMeshRepoThread::loadMeshSkinInfo(const LLUUID& mesh_id) @@ -800,6 +817,42 @@ std::string LLMeshRepoThread::constructUrl(LLUUID mesh_id) return http_url; } +// May only be called by repo thread +LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, + size_t offset, + size_t len, + LLCore::HttpHandler * handler) +{ + LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + if (len < LARGE_MESH_FETCH_THRESHOLD) + { + handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, + mHttpPriority, + url, + offset, + len, + mHttpOptions, + mHttpHeaders, + handler); + ++mHttpGetCount; + } + else + { + handle = mHttpRequest->requestGetByteRange(mHttpLargePolicyClass, + mHttpPriority, + url, + offset, + len, + mHttpLargeOptions, + mHttpHeaders, + handler); + ++mHttpLargeGetCount; + } + return handle; +} + + bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) { //protected by mMutex @@ -863,14 +916,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) { LLMeshSkinInfoHandler * handler = new LLMeshSkinInfoHandler(mesh_id, offset, size); // LL_WARNS("Mesh") << "MESH: Issuing Skin Info Request" << LL_ENDL; - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, - 0, // *TODO: Get better priority value - http_url, - offset, - size, - mHttpOptions, - mHttpHeaders, - handler); + LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -959,14 +1005,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) { LLMeshDecompositionHandler * handler = new LLMeshDecompositionHandler(mesh_id, offset, size); // LL_WARNS("Mesh") << "MESH: Issuing Decomp Request" << LL_ENDL; - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, - 0, // *TODO: Get better priority value - http_url, - offset, - size, - mHttpOptions, - mHttpHeaders, - handler); + LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -1054,14 +1093,7 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) { LLMeshPhysicsShapeHandler * handler = new LLMeshPhysicsShapeHandler(mesh_id, offset, size); // LL_WARNS("Mesh") << "MESH: Issuing Physics Shape Request" << LL_ENDL; - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, - 0, // *TODO: Get better priority value - http_url, - offset, - size, - mHttpOptions, - mHttpHeaders, - handler); + LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -1154,14 +1186,7 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params); // LL_WARNS("Mesh") << "MESH: Issuing Request" << LL_ENDL; - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, - 0, // *TODO: Get better priority value - http_url, - 0, - MESH_HEADER_SIZE, - mHttpOptions, - mHttpHeaders, - handler); + LLCore::HttpHandle handle = getByteRange(http_url, 0, MESH_HEADER_SIZE, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -1241,14 +1266,7 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, { LLMeshLODHandler * handler = new LLMeshLODHandler(mesh_params, lod, offset, size); // LL_WARNS("Mesh") << "MESH: Issuing LOD Request" << LL_ENDL; - LLCore::HttpHandle handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, - 0, // *TODO: Get better priority value - http_url, - offset, - size, - mHttpOptions, - mHttpHeaders, - handler); + LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -2537,9 +2555,14 @@ S32 LLMeshRepository::loadMesh(LLVOVolume* vobj, const LLVolumeParams& mesh_para void LLMeshRepository::notifyLoadedMeshes() { //called from main thread - - LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests"); - + // *FIXME: Scaling down the setting by a factor of 4 for now to reflect + // target goal. May want to rename the setting before release. + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests") / 4; + LLMeshRepoThread::sRequestHighWater = llmax(50 * S32(LLMeshRepoThread::sMaxConcurrentRequests), + REQUEST_HIGH_WATER_MIN); + LLMeshRepoThread::sRequestLowWater = llmax(LLMeshRepoThread::sRequestLowWater / 2, + REQUEST_LOW_WATER_MIN); + //clean up completed upload threads for (std::vector::iterator iter = mUploads.begin(); iter != mUploads.end(); ) { @@ -2617,7 +2640,7 @@ void LLMeshRepository::notifyLoadedMeshes() //call completed callbacks on finished decompositions mDecompThread->notifyCompleted(); - if (!mThread->mWaiting) + if (!mThread->mWaiting && mPendingRequests.empty()) { //curl thread is churning, wait for it to go idle return; } @@ -2644,47 +2667,55 @@ void LLMeshRepository::notifyLoadedMeshes() mUploadErrorQ.pop(); } - S32 push_count = LLMeshRepoThread::sMaxConcurrentRequests-(LLMeshRepoThread::sActiveHeaderRequests+LLMeshRepoThread::sActiveLODRequests); - - if (push_count > 0) + S32 active_count = LLMeshRepoThread::sActiveHeaderRequests + LLMeshRepoThread::sActiveLODRequests; + if (active_count < LLMeshRepoThread::sRequestLowWater) { - //calculate "score" for pending requests - - //create score map - std::map score_map; + S32 push_count = LLMeshRepoThread::sRequestHighWater - active_count; - for (U32 i = 0; i < 4; ++i) + if (mPendingRequests.size() > push_count) { - for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) + // More requests than the high-water limit allows so + // sort and forward the most important. + + //calculate "score" for pending requests + + //create score map + std::map score_map; + + for (U32 i = 0; i < 4; ++i) { - F32 max_score = 0.f; - for (std::set::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) + for (mesh_load_map::iterator iter = mLoadingMeshes[i].begin(); iter != mLoadingMeshes[i].end(); ++iter) { - LLViewerObject* object = gObjectList.findObject(*obj_iter); - - if (object) + F32 max_score = 0.f; + for (std::set::iterator obj_iter = iter->second.begin(); obj_iter != iter->second.end(); ++obj_iter) { - LLDrawable* drawable = object->mDrawable; - if (drawable) + LLViewerObject* object = gObjectList.findObject(*obj_iter); + + if (object) { - F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); - max_score = llmax(max_score, cur_score); + LLDrawable* drawable = object->mDrawable; + if (drawable) + { + F32 cur_score = drawable->getRadius()/llmax(drawable->mDistanceWRTCamera, 1.f); + max_score = llmax(max_score, cur_score); + } } } - } - score_map[iter->first.getSculptID()] = max_score; + score_map[iter->first.getSculptID()] = max_score; + } } - } - //set "score" for pending requests - for (std::vector::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) - { - iter->mScore = score_map[iter->mMeshParams.getSculptID()]; - } + //set "score" for pending requests + for (std::vector::iterator iter = mPendingRequests.begin(); iter != mPendingRequests.end(); ++iter) + { + iter->mScore = score_map[iter->mMeshParams.getSculptID()]; + } - //sort by "score" - std::sort(mPendingRequests.begin(), mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); + //sort by "score" + std::partial_sort(mPendingRequests.begin(), mPendingRequests.begin() + push_count, + mPendingRequests.end(), LLMeshRepoThread::CompareScoreGreater()); + } while (!mPendingRequests.empty() && push_count > 0) { diff --git a/indra/newview/llmeshrepository.h b/indra/newview/llmeshrepository.h index 62f81ce9e2..0dca29e7d4 100755 --- a/indra/newview/llmeshrepository.h +++ b/indra/newview/llmeshrepository.h @@ -224,13 +224,14 @@ public: static S32 sActiveHeaderRequests; static S32 sActiveLODRequests; static U32 sMaxConcurrentRequests; + static S32 sRequestLowWater; + static S32 sRequestHighWater; - LLCurlRequest* mCurlRequest; LLMutex* mMutex; LLMutex* mHeaderMutex; LLCondition* mSignal; - bool mWaiting; + volatile bool mWaiting; //map of known mesh headers typedef std::map mesh_header_map; @@ -324,8 +325,11 @@ public: // llcorehttp library interface objects. LLCore::HttpRequest * mHttpRequest; LLCore::HttpOptions * mHttpOptions; + LLCore::HttpOptions * mHttpLargeOptions; LLCore::HttpHeaders * mHttpHeaders; LLCore::HttpRequest::policy_t mHttpPolicyClass; + LLCore::HttpRequest::policy_t mHttpLargePolicyClass; + LLCore::HttpRequest::priority_t mHttpPriority; typedef std::set http_request_set; http_request_set mHttpRequestSet; // Outstanding HTTP requests @@ -373,6 +377,19 @@ public: static void incActiveHeaderRequests(); static void decActiveHeaderRequests(); +private: + // Issue a GET request to a URL with 'Range' header using + // the correct policy class and other attributes. If an invalid + // handle is returned, the request failed and caller must retry + // or dispose of handler. + // + // Threads: Repo thread only + LLCore::HttpHandle getByteRange(const std::string & url, size_t offset, size_t len, + LLCore::HttpHandler * handler); + +private: + U32 mHttpGetCount; + U32 mHttpLargeGetCount; }; class LLMeshUploadThread : public LLThread diff --git a/indra/newview/lltexturefetch.cpp b/indra/newview/lltexturefetch.cpp index be5fde9e2b..d934ef9dc4 100755 --- a/indra/newview/lltexturefetch.cpp +++ b/indra/newview/lltexturefetch.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2000&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -2410,7 +2410,7 @@ LLTextureFetch::LLTextureFetch(LLTextureCache* cache, LLImageDecodeThread* image mHttpHeaders->mHeaders.push_back("Accept: image/x-j2c"); mHttpMetricsHeaders = new LLCore::HttpHeaders; mHttpMetricsHeaders->mHeaders.push_back("Content-Type: application/llsd+xml"); - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicyDefault(); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_TEXTURE); } LLTextureFetch::~LLTextureFetch() -- cgit v1.3 From d6cbcd591aea32357d50b266efe8a95754302cbf Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Thu, 20 Jun 2013 19:18:39 -0400 Subject: SH-4257 Preparation for a new cap grant: GetMesh2 Mesh repo is using three policy classes now: one for large objects, one for GetMesh2 regions, one for GetMesh regions. It's also detecting the presence of the cap and using the correct class. Class initialization cleaned up significantly in llappcorehttp using data-directed code. Pulled in the changes to HttpHeader done for sunshine-internal then did a refactoring pass on the header callback which now uses a unified approach to clean up and deliver header information to all interested parties. Added support for using Retry-After header information on 503 retries. --- indra/llcorehttp/_httpinternal.h | 6 +- indra/llcorehttp/_httpoperation.cpp | 4 +- indra/llcorehttp/_httpoprequest.cpp | 244 ++++++++++++++++++++++++++-------- indra/llcorehttp/_httpoprequest.h | 4 + indra/llcorehttp/_httppolicy.cpp | 12 +- indra/llcorehttp/_httppolicyclass.cpp | 4 +- indra/llcorehttp/_httpservice.cpp | 3 - indra/llcorehttp/httpoptions.cpp | 8 +- indra/llcorehttp/httpoptions.h | 8 +- indra/newview/llappcorehttp.cpp | 10 +- indra/newview/llappcorehttp.h | 3 +- indra/newview/llmeshrepository.cpp | 55 +++++--- indra/newview/llmeshrepository.h | 8 +- 13 files changed, 275 insertions(+), 94 deletions(-) (limited to 'indra/newview/llappcorehttp.cpp') diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index d60996756f..f085ca3b91 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -98,7 +98,7 @@ namespace LLCore // Maxium number of policy classes that can be defined. // *TODO: Currently limited to the default class + 1, extend. -const int HTTP_POLICY_CLASS_LIMIT = 4; +const int HTTP_POLICY_CLASS_LIMIT = 8; // Debug/informational tracing. Used both // as a global option and in per-request traces. @@ -138,6 +138,10 @@ const int HTTP_CONNECTION_LIMIT_DEFAULT = 8; const int HTTP_CONNECTION_LIMIT_MIN = 1; const int HTTP_CONNECTION_LIMIT_MAX = 256; +// Miscellaneous defaults +const long HTTP_PIPELINING_DEFAULT = 0L; +const bool HTTP_USE_RETRY_AFTER_DEFAULT = true; + // Tuning parameters // Time worker thread sleeps after a pass through the diff --git a/indra/llcorehttp/_httpoperation.cpp b/indra/llcorehttp/_httpoperation.cpp index 5cf5bc5930..7acd728bbd 100755 --- a/indra/llcorehttp/_httpoperation.cpp +++ b/indra/llcorehttp/_httpoperation.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -53,7 +53,7 @@ HttpOperation::HttpOperation() mUserHandler(NULL), mReqPolicy(HttpRequest::DEFAULT_POLICY_ID), mReqPriority(0U), - mTracing(0) + mTracing(HTTP_TRACE_OFF) { mMetricCreated = totalTime(); } diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index a4c0a12fdc..8cb7fee701 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -64,6 +64,15 @@ int parse_content_range_header(char * buffer, unsigned int * last, unsigned int * length); +// Similar for Retry-After headers. Only parses the delta form +// of the header, HTTP time formats aren't interesting for client +// purposes. +// +// @return 0 if successfully parsed and seconds time delta +// returned in time argument. +// +int parse_retry_after_header(char * buffer, int * time); + // Take data from libcurl's CURLOPT_DEBUGFUNCTION callback and // escape and format it for a tracing line in logging. Absolutely @@ -74,14 +83,12 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line); -// OS-neutral string comparisons of various types -int os_strncasecmp(const char *s1, const char *s2, size_t n); -int os_strcasecmp(const char *s1, const char *s2); -char * os_strtok_r(char *str, const char *delim, char **saveptr); - - -static const char * const hdr_whitespace(" \t"); -static const char * const hdr_separator(": \t"); +// OS-neutral string comparisons of various types. +int os_strcasecmp(const char * s1, const char * s2); +char * os_strtok_r(char * str, const char * delim, char ** saveptr); +char * os_strtrim(char * str); +char * os_strltrim(char * str); +void os_strlower(char * str); } // end anonymous namespace @@ -104,6 +111,8 @@ HttpOpRequest::HttpOpRequest() mCurlService(NULL), mCurlHeaders(NULL), mCurlBodyPos(0), + mCurlTemp(NULL), + mCurlTempLen(0), mReplyBody(NULL), mReplyOffset(0), mReplyLength(0), @@ -154,6 +163,10 @@ HttpOpRequest::~HttpOpRequest() mCurlHeaders = NULL; } + delete [] mCurlTemp; + mCurlTemp = NULL; + mCurlTempLen = 0; + if (mReplyBody) { mReplyBody->release(); @@ -207,6 +220,11 @@ void HttpOpRequest::stageFromActive(HttpService * service) mCurlHeaders = NULL; } + // Also not needed on the other side + delete [] mCurlTemp; + mCurlTemp = NULL; + mCurlTempLen = 0; + addAsReply(); } @@ -335,6 +353,10 @@ void HttpOpRequest::setupCommon(HttpRequest::policy_t policy_id, { mProcFlags |= PF_SAVE_HEADERS; } + if (options->getUseRetryAfter()) + { + mProcFlags |= PF_USE_RETRY_AFTER; + } mPolicyRetryLimit = options->getRetries(); mPolicyRetryLimit = llclamp(mPolicyRetryLimit, HTTP_RETRY_COUNT_MIN, HTTP_RETRY_COUNT_MAX); mTracing = (std::max)(mTracing, llclamp(options->getTrace(), HTTP_TRACE_MIN, HTTP_TRACE_MAX)); @@ -549,7 +571,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) } curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mCurlHeaders); - if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS)) + if (mProcFlags & (PF_SCAN_RANGE_HEADER | PF_SAVE_HEADERS | PF_USE_RETRY_AFTER)) { curl_easy_setopt(mCurlHandle, CURLOPT_HEADERFUNCTION, headerCallback); curl_easy_setopt(mCurlHandle, CURLOPT_HEADERDATA, this); @@ -610,10 +632,9 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi { static const char status_line[] = "HTTP/"; static const size_t status_line_len = sizeof(status_line) - 1; - - static const char con_ran_line[] = "content-range:"; - static const size_t con_ran_line_len = sizeof(con_ran_line) - 1; - + static const char con_ran_line[] = "content-range"; + static const char con_retry_line[] = "retry-after"; + HttpOpRequest * op(static_cast(userdata)); const size_t hdr_size(size * nmemb); @@ -627,6 +648,7 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi op->mReplyOffset = 0; op->mReplyLength = 0; op->mReplyFullLength = 0; + op->mReplyRetryAfter = 0; op->mStatus = HttpStatus(); if (op->mReplyHeaders) { @@ -645,6 +667,53 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi --wanted_hdr_size; } } + + // Copy and normalize header fragments for the following + // stages. Would like to modify the data in-place but that + // may not be allowed and we need one byte extra for NUL. + // At the end of this we will have: + // + // If ':' present in header: + // 1. name points to text to left of colon which + // will be ascii lower-cased and left and right + // trimmed of whitespace. + // 2. value points to text to right of colon which + // will be left trimmed of whitespace. + // Otherwise: + // 1. name points to header which will be left + // trimmed of whitespace. + // 2. value is NULL + // Any non-NULL pointer may point to a zero-length string. + // + if (wanted_hdr_size >= op->mCurlTempLen) + { + delete [] op->mCurlTemp; + op->mCurlTempLen = 2 * wanted_hdr_size + 1; + op->mCurlTemp = new char [op->mCurlTempLen]; + } + memcpy(op->mCurlTemp, hdr_data, wanted_hdr_size); + op->mCurlTemp[wanted_hdr_size] = '\0'; + char * name(op->mCurlTemp); + char * value(strchr(name, ':')); + if (value) + { + *value++ = '\0'; + os_strlower(name); + name = os_strtrim(name); + value = os_strltrim(value); + } + else + { + // Doesn't look well-formed, do minimal normalization on it + name = os_strltrim(name); + } + + // Normalized, now reject headers with empty names. + if (! *name) + { + // No use continuing + return hdr_size; + } // Save header if caller wants them in the response if (is_header && op->mProcFlags & PF_SAVE_HEADERS) @@ -654,43 +723,53 @@ size_t HttpOpRequest::headerCallback(void * data, size_t size, size_t nmemb, voi { op->mReplyHeaders = new HttpHeaders; } - op->mReplyHeaders->appendNormal(hdr_data, wanted_hdr_size); + op->mReplyHeaders->append(name, value ? value : ""); } + // From this point, header-specific processors are free to + // modify the header value. + // Detect and parse 'Content-Range' headers - if (is_header && op->mProcFlags & PF_SCAN_RANGE_HEADER) + if (is_header + && op->mProcFlags & PF_SCAN_RANGE_HEADER + && value && *value + && ! strcmp(name, con_ran_line)) { - char hdr_buffer[128]; // Enough for a reasonable header - size_t frag_size((std::min)(wanted_hdr_size, sizeof(hdr_buffer) - 1)); - - memcpy(hdr_buffer, hdr_data, frag_size); - hdr_buffer[frag_size] = '\0'; - if (frag_size > con_ran_line_len && - ! os_strncasecmp(hdr_buffer, con_ran_line, con_ran_line_len)) + unsigned int first(0), last(0), length(0); + int status; + + if (! (status = parse_content_range_header(value, &first, &last, &length))) + { + // Success, record the fragment position + op->mReplyOffset = first; + op->mReplyLength = last - first + 1; + op->mReplyFullLength = length; + } + else if (-1 == status) { - unsigned int first(0), last(0), length(0); - int status; + // Response is badly formed and shouldn't be accepted + op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); + } + else + { + // Ignore the unparsable. + LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" + << std::string(hdr_data, wanted_hdr_size) + << "'. Ignoring." + << LL_ENDL; + } + } - if (! (status = parse_content_range_header(hdr_buffer, &first, &last, &length))) - { - // Success, record the fragment position - op->mReplyOffset = first; - op->mReplyLength = last - first + 1; - op->mReplyFullLength = length; - } - else if (-1 == status) - { - // Response is badly formed and shouldn't be accepted - op->mStatus = HttpStatus(HttpStatus::LLCORE, HE_INV_CONTENT_RANGE_HDR); - } - else - { - // Ignore the unparsable. - LL_INFOS_ONCE("CoreHttp") << "Problem parsing odd Content-Range header: '" - << std::string(hdr_data, frag_size) - << "'. Ignoring." - << LL_ENDL; - } + // Detect and parse 'Retry-After' headers + if (is_header + && op->mProcFlags & PF_USE_RETRY_AFTER + && value && *value + && ! strcmp(name, con_retry_line)) + { + int time(0); + if (! parse_retry_after_header(value, &time)) + { + op->mReplyRetryAfter = time; } } @@ -805,14 +884,16 @@ int parse_content_range_header(char * buffer, unsigned int * last, unsigned int * length) { + static const char * const hdr_whitespace(" \t"); + char * tok_state(NULL), * tok(NULL); bool match(true); - if (! os_strtok_r(buffer, hdr_separator, &tok_state)) + if (! (tok = os_strtok_r(buffer, hdr_whitespace, &tok_state))) match = false; - if (match && (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) - match = 0 == os_strcasecmp("bytes", tok); - if (match && ! (tok = os_strtok_r(NULL, " \t", &tok_state))) + else + match = (0 == os_strcasecmp("bytes", tok)); + if (match && ! (tok = os_strtok_r(NULL, hdr_whitespace, &tok_state))) match = false; if (match) { @@ -851,6 +932,25 @@ int parse_content_range_header(char * buffer, } +int parse_retry_after_header(char * buffer, int * time) +{ + char * endptr(buffer); + long lcl_time(strtol(buffer, &endptr, 10)); + if (*endptr == '\0' && endptr != buffer && lcl_time > 0) + { + *time = lcl_time; + return 0; + } + + // Could attempt to parse HTTP time here but we're not really + // interested in it. Scheduling based on wallclock time on + // user hardware will lead to tears. + + // Header is there but badly/unexpectedly formed, try to ignore it. + return 1; +} + + void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::string & safe_line) { std::string out; @@ -887,15 +987,6 @@ void escape_libcurl_debug_data(char * buffer, size_t len, bool scrub, std::strin } -int os_strncasecmp(const char *s1, const char *s2, size_t n) -{ -#if LL_WINDOWS - return _strnicmp(s1, s2, n); -#else - return strncasecmp(s1, s2, n); -#endif // LL_WINDOWS -} - int os_strcasecmp(const char *s1, const char *s2) { @@ -917,6 +1008,45 @@ char * os_strtok_r(char *str, const char *delim, char ** savestate) } +void os_strlower(char * str) +{ + for (char c(0); (c = *str); ++str) + { + *str = tolower(c); + } +} + + +char * os_strtrim(char * lstr) +{ + while (' ' == *lstr || '\t' == *lstr) + { + ++lstr; + } + if (*lstr) + { + for (char * rstr(lstr + strlen(lstr)); *--rstr;) + { + if (' ' == *rstr || '\t' == *rstr) + { + *rstr = '\0'; + } + } + } + return lstr; +} + + +char * os_strltrim(char * lstr) +{ + while (' ' == *lstr || '\t' == *lstr) + { + ++lstr; + } + return lstr; +} + + } // end anonymous namespace diff --git a/indra/llcorehttp/_httpoprequest.h b/indra/llcorehttp/_httpoprequest.h index 831e5bebf7..2e737cf1cc 100755 --- a/indra/llcorehttp/_httpoprequest.h +++ b/indra/llcorehttp/_httpoprequest.h @@ -158,6 +158,7 @@ protected: unsigned int mProcFlags; static const unsigned int PF_SCAN_RANGE_HEADER = 0x00000001U; static const unsigned int PF_SAVE_HEADERS = 0x00000002U; + static const unsigned int PF_USE_RETRY_AFTER = 0x00000004U; public: // Request data @@ -175,6 +176,8 @@ public: HttpService * mCurlService; curl_slist * mCurlHeaders; size_t mCurlBodyPos; + char * mCurlTemp; // Scratch buffer for header processing + size_t mCurlTempLen; // Result data HttpStatus mStatus; @@ -184,6 +187,7 @@ public: size_t mReplyFullLength; HttpHeaders * mReplyHeaders; std::string mReplyConType; + int mReplyRetryAfter; // Policy data int mPolicyRetries; diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 54c9c6bb1b..5f303dd0fe 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -160,8 +160,12 @@ void HttpPolicy::retryOp(HttpOpRequest * op) const HttpTime now(totalTime()); const int policy_class(op->mReqPolicy); - - const HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + HttpTime delta(retry_deltas[llclamp(op->mPolicyRetries, 0, delta_max)]); + + if (op->mReplyRetryAfter > 0 && op->mReplyRetryAfter < 30) + { + delta = op->mReplyRetryAfter * U64L(1000000); + } op->mPolicyRetryAt = now + delta; ++op->mPolicyRetries; if (error_503 == op->mStatus) @@ -170,10 +174,10 @@ void HttpPolicy::retryOp(HttpOpRequest * op) } LL_WARNS("CoreHttp") << "HTTP request " << static_cast(op) << " retry " << op->mPolicyRetries - << " scheduled for +" << (delta / HttpTime(1000)) + << " scheduled in " << (delta / HttpTime(1000)) << " mS. Status: " << op->mStatus.toHex() << LL_ENDL; - if (op->mTracing > 0) + if (op->mTracing > HTTP_TRACE_OFF) { LL_INFOS("CoreHttp") << "TRACE, ToRetryQueue, Handle: " << static_cast(op) diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp index a23b81322c..1a55ab1ac6 100755 --- a/indra/llcorehttp/_httppolicyclass.cpp +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -37,7 +37,7 @@ HttpPolicyClass::HttpPolicyClass() : mSetMask(0UL), mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), - mPipelining(0) + mPipelining(HTTP_PIPELINING_DEFAULT) {} diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 0825888d0f..0821401289 100755 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -55,9 +55,6 @@ HttpService::HttpService() { // Create the default policy class HttpPolicyClass pol_class; - pol_class.set(HttpRequest::CP_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); - pol_class.set(HttpRequest::CP_PER_HOST_CONNECTION_LIMIT, HTTP_CONNECTION_LIMIT_DEFAULT); - pol_class.set(HttpRequest::CP_ENABLE_PIPELINING, 0L); mPolicyClasses.push_back(pol_class); } diff --git a/indra/llcorehttp/httpoptions.cpp b/indra/llcorehttp/httpoptions.cpp index 4dcd862ca4..5bf1ecb4a5 100755 --- a/indra/llcorehttp/httpoptions.cpp +++ b/indra/llcorehttp/httpoptions.cpp @@ -39,7 +39,8 @@ HttpOptions::HttpOptions() mTracing(HTTP_TRACE_OFF), mTimeout(HTTP_REQUEST_TIMEOUT_DEFAULT), mTransferTimeout(HTTP_REQUEST_XFER_TIMEOUT_DEFAULT), - mRetries(HTTP_RETRY_COUNT_DEFAULT) + mRetries(HTTP_RETRY_COUNT_DEFAULT), + mUseRetryAfter(HTTP_USE_RETRY_AFTER_DEFAULT) {} @@ -76,5 +77,10 @@ void HttpOptions::setRetries(unsigned int retries) mRetries = retries; } +void HttpOptions::setUseRetryAfter(bool use_retry) +{ + mUseRetryAfter = use_retry; +} + } // end namespace LLCore diff --git a/indra/llcorehttp/httpoptions.h b/indra/llcorehttp/httpoptions.h index 623d71d3e6..04531425d8 100755 --- a/indra/llcorehttp/httpoptions.h +++ b/indra/llcorehttp/httpoptions.h @@ -98,13 +98,19 @@ public: return mRetries; } + void setUseRetryAfter(bool use_retry); + bool getUseRetryAfter() const + { + return mUseRetryAfter; + } + protected: bool mWantHeaders; int mTracing; unsigned int mTimeout; unsigned int mTransferTimeout; unsigned int mRetries; - + bool mUseRetryAfter; }; // end class HttpOptions diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index b601b31d21..2467c02d4d 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -72,10 +72,16 @@ void LLAppCoreHttp::init() "texture fetch" }, { - AP_MESH, 8, 1, 32, 4, + // *FIXME: Should become 32, 1, 32, 1 before release + AP_MESH1, 8, 1, 32, 4, "MeshMaxConcurrentRequests", "mesh fetch" }, + { + AP_MESH2, 8, 1, 32, 4, + "MeshMaxConcurrentRequests", + "mesh2 fetch" + }, { AP_LARGE_MESH, 2, 1, 8, 1, "", @@ -171,6 +177,8 @@ void LLAppCoreHttp::init() } // Set it and report + // *TODO: These are intended to be per-host limits when we can + // support that in llcorehttp/libcurl. LLCore::HttpStatus status; status = LLCore::HttpRequest::setPolicyClassOption(mPolicies[policy], LLCore::HttpRequest::CP_CONNECTION_LIMIT, diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 532e1f5cb0..4a14c35966 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -47,7 +47,8 @@ public: { AP_DEFAULT, AP_TEXTURE, - AP_MESH, + AP_MESH1, + AP_MESH2, AP_LARGE_MESH, AP_UPLOADS, AP_COUNT // Must be last diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index 221a797fc7..1cda0b6a71 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -560,6 +560,7 @@ LLMeshRepoThread::LLMeshRepoThread() mHttpLargeOptions(NULL), mHttpHeaders(NULL), mHttpPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), + mHttpLegacyPolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), mHttpLargePolicyClass(LLCore::HttpRequest::DEFAULT_POLICY_ID), mHttpPriority(0), mHttpGetCount(0U), @@ -574,8 +575,9 @@ LLMeshRepoThread::LLMeshRepoThread() mHttpLargeOptions->setTransferTimeout(LARGE_MESH_XFER_TIMEOUT); mHttpHeaders = new LLCore::HttpHeaders; mHttpHeaders->append("Accept", "application/vnd.ll.mesh"); - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH); - mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_LARGE_MESH); + mHttpPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH2); + mHttpLegacyPolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_MESH1); + mHttpLargePolicyClass = LLAppViewer::instance()->getAppCoreHttp().getPolicy(LLAppCoreHttp::AP_LARGE_MESH); } @@ -795,13 +797,22 @@ void LLMeshRepoThread::loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod) } //static -std::string LLMeshRepoThread::constructUrl(LLUUID mesh_id) +std::string LLMeshRepoThread::constructUrl(LLUUID mesh_id, int * cap_version) { + int version(1); std::string http_url; if (gAgent.getRegion()) { - http_url = gMeshRepo.mGetMeshCapability; + if (! gMeshRepo.mGetMesh2Capability.empty()) + { + version = 2; + http_url = gMeshRepo.mGetMesh2Capability; + } + else + { + http_url = gMeshRepo.mGetMeshCapability; + } } if (!http_url.empty()) @@ -814,20 +825,22 @@ std::string LLMeshRepoThread::constructUrl(LLUUID mesh_id) llwarns << "Current region does not have GetMesh capability! Cannot load " << mesh_id << ".mesh" << llendl; } + *cap_version = version; return http_url; } // May only be called by repo thread -LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, - size_t offset, - size_t len, +LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int cap_version, + size_t offset, size_t len, LLCore::HttpHandler * handler) { LLCore::HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); if (len < LARGE_MESH_FETCH_THRESHOLD) { - handle = mHttpRequest->requestGetByteRange(mHttpPolicyClass, + handle = mHttpRequest->requestGetByteRange((2 == cap_version + ? mHttpPolicyClass + : mHttpLegacyPolicyClass), mHttpPriority, url, offset, @@ -911,12 +924,13 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) } //reading from VFS failed for whatever reason, fetch from sim - std::string http_url = constructUrl(mesh_id); + int cap_version(1); + std::string http_url = constructUrl(mesh_id, &cap_version); if (!http_url.empty()) { LLMeshSkinInfoHandler * handler = new LLMeshSkinInfoHandler(mesh_id, offset, size); // LL_WARNS("Mesh") << "MESH: Issuing Skin Info Request" << LL_ENDL; - LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -1000,12 +1014,13 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) } //reading from VFS failed for whatever reason, fetch from sim - std::string http_url = constructUrl(mesh_id); + int cap_version(1); + std::string http_url = constructUrl(mesh_id, &cap_version); if (!http_url.empty()) { LLMeshDecompositionHandler * handler = new LLMeshDecompositionHandler(mesh_id, offset, size); // LL_WARNS("Mesh") << "MESH: Issuing Decomp Request" << LL_ENDL; - LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -1088,12 +1103,13 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) } //reading from VFS failed for whatever reason, fetch from sim - std::string http_url = constructUrl(mesh_id); + int cap_version(1); + std::string http_url = constructUrl(mesh_id, &cap_version); if (!http_url.empty()) { LLMeshPhysicsShapeHandler * handler = new LLMeshPhysicsShapeHandler(mesh_id, offset, size); // LL_WARNS("Mesh") << "MESH: Issuing Physics Shape Request" << LL_ENDL; - LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -1177,7 +1193,8 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c //either cache entry doesn't exist or is corrupt, request header from simulator bool retval = true ; - std::string http_url = constructUrl(mesh_params.getSculptID()); + int cap_version(1); + std::string http_url = constructUrl(mesh_params.getSculptID(), &cap_version); if (!http_url.empty()) { //grab first 4KB if we're going to bother with a fetch. Cache will prevent future fetches if a full mesh fits @@ -1186,7 +1203,7 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c LLMeshHeaderHandler * handler = new LLMeshHeaderHandler(mesh_params); // LL_WARNS("Mesh") << "MESH: Issuing Request" << LL_ENDL; - LLCore::HttpHandle handle = getByteRange(http_url, 0, MESH_HEADER_SIZE, handler); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, 0, MESH_HEADER_SIZE, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -1261,12 +1278,13 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, } //reading from VFS failed for whatever reason, fetch from sim - std::string http_url = constructUrl(mesh_id); + int cap_version(1); + std::string http_url = constructUrl(mesh_id, &cap_version); if (!http_url.empty()) { LLMeshLODHandler * handler = new LLMeshLODHandler(mesh_params, lod, offset, size); // LL_WARNS("Mesh") << "MESH: Issuing LOD Request" << LL_ENDL; - LLCore::HttpHandle handle = getByteRange(http_url, offset, size, handler); + LLCore::HttpHandle handle = getByteRange(http_url, cap_version, offset, size, handler); if (LLCORE_HTTP_HANDLE_INVALID == handle) { // *TODO: Better error message @@ -2653,6 +2671,7 @@ void LLMeshRepository::notifyLoadedMeshes() { region_name = gAgent.getRegion()->getName(); mGetMeshCapability = gAgent.getRegion()->getCapability("GetMesh"); + mGetMesh2Capability = gAgent.getRegion()->getCapability("GetMesh2"); } } diff --git a/indra/newview/llmeshrepository.h b/indra/newview/llmeshrepository.h index 0dca29e7d4..74690e5a2a 100755 --- a/indra/newview/llmeshrepository.h +++ b/indra/newview/llmeshrepository.h @@ -328,13 +328,14 @@ public: LLCore::HttpOptions * mHttpLargeOptions; LLCore::HttpHeaders * mHttpHeaders; LLCore::HttpRequest::policy_t mHttpPolicyClass; + LLCore::HttpRequest::policy_t mHttpLegacyPolicyClass; LLCore::HttpRequest::policy_t mHttpLargePolicyClass; LLCore::HttpRequest::priority_t mHttpPriority; typedef std::set http_request_set; http_request_set mHttpRequestSet; // Outstanding HTTP requests - static std::string constructUrl(LLUUID mesh_id); + static std::string constructUrl(LLUUID mesh_id, int * cap_version); LLMeshRepoThread(); ~LLMeshRepoThread(); @@ -384,7 +385,8 @@ private: // or dispose of handler. // // Threads: Repo thread only - LLCore::HttpHandle getByteRange(const std::string & url, size_t offset, size_t len, + LLCore::HttpHandle getByteRange(const std::string & url, int cap_version, + size_t offset, size_t len, LLCore::HttpHandler * handler); private: @@ -595,7 +597,7 @@ public: void updateInventory(inventory_data data); std::string mGetMeshCapability; - + std::string mGetMesh2Capability; }; extern LLMeshRepository gMeshRepo; -- cgit v1.3 From a12d5d7c6d8d816512ec9fc1355b37ce38a89cce Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 28 Jun 2013 20:08:29 -0400 Subject: SH-4312 Clumsy configuration coordination between mesh and corehttp Taught llappcorehttp to register signals on the settings values that chagne behavior. Have initialization and settings changes sweep through settings and change them. Dynamic changes are tried but have no effect (produce a warning message) as dynamic settings still aren't supported but the plumbing is now connected. Just need to change llcorehttp. Bounced the 'teleport started' signal around and it ended up back where it started with some cleanup. This is making me less angry... --- indra/newview/llappcorehttp.cpp | 210 +++++++++++++++++++++++-------------- indra/newview/llappcorehttp.h | 9 +- indra/newview/llmeshrepository.cpp | 30 ++---- 3 files changed, 145 insertions(+), 104 deletions(-) (limited to 'indra/newview/llappcorehttp.cpp') diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 2467c02d4d..70e349e33d 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -28,10 +28,52 @@ #include "llappcorehttp.h" +#include "llappviewer.h" #include "llviewercontrol.h" const F64 LLAppCoreHttp::MAX_THREAD_WAIT_TIME(10.0); +static const struct +{ + LLAppCoreHttp::EAppPolicy mPolicy; + U32 mDefault; + U32 mMin; + U32 mMax; + U32 mDivisor; + std::string mKey; + const char * mUsage; +} init_data[] = // Default and dynamic values for classes +{ + { + LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 1, + "TextureFetchConcurrency", + "texture fetch" + }, + { + LLAppCoreHttp::AP_MESH1, 32, 1, 128, 1, + "MeshMaxConcurrentRequests", + "mesh fetch" + }, + { + LLAppCoreHttp::AP_MESH2, 8, 1, 32, 4, + "MeshMaxConcurrentRequests", + "mesh2 fetch" + }, + { + LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 1, + "", + "large mesh fetch" + }, + { + LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 1, + "", + "asset upload" + } +}; + +static void teleport_started(); +static void setting_changed(); + LLAppCoreHttp::LLAppCoreHttp() : mRequest(NULL), @@ -42,6 +84,7 @@ LLAppCoreHttp::LLAppCoreHttp() for (int i(0); i < LL_ARRAY_SIZE(mPolicies); ++i) { mPolicies[i] = LLCore::HttpRequest::DEFAULT_POLICY_ID; + mSettings[i] = 0U; } } @@ -55,45 +98,6 @@ LLAppCoreHttp::~LLAppCoreHttp() void LLAppCoreHttp::init() { - static const struct - { - EAppPolicy mPolicy; - U32 mDefault; - U32 mMin; - U32 mMax; - U32 mDivisor; - std::string mKey; - const char * mUsage; - } init_data[] = // Default and dynamic values for classes - { - { - AP_TEXTURE, 8, 1, 12, 1, - "TextureFetchConcurrency", - "texture fetch" - }, - { - // *FIXME: Should become 32, 1, 32, 1 before release - AP_MESH1, 8, 1, 32, 4, - "MeshMaxConcurrentRequests", - "mesh fetch" - }, - { - AP_MESH2, 8, 1, 32, 4, - "MeshMaxConcurrentRequests", - "mesh2 fetch" - }, - { - AP_LARGE_MESH, 2, 1, 8, 1, - "", - "large mesh fetch" - }, - { - AP_UPLOADS, 2, 1, 8, 1, - "", - "asset upload" - } - }; - LLCore::HttpStatus status = LLCore::HttpRequest::createService(); if (! status) { @@ -110,14 +114,12 @@ void LLAppCoreHttp::init() << LL_ENDL; } - // Establish HTTP Proxy. "LLProxy" is a special string which directs - // the code to use LLProxy::applyProxySettings() to establish any - // HTTP or SOCKS proxy for http operations. + // Establish HTTP Proxy, if desired. status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); if (! status) { - LL_ERRS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " << status.toString() - << LL_ENDL; + LL_WARNS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " << status.toString() + << LL_ENDL; } // Tracing levels for library & libcurl (note that 2 & 3 are beyond spammy): @@ -137,7 +139,6 @@ void LLAppCoreHttp::init() mPolicies[AP_DEFAULT] = LLCore::HttpRequest::DEFAULT_POLICY_ID; // Setup additional policies based on table and some special rules - // *TODO: Make these configurations dynamic later for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { const EAppPolicy policy(init_data[i].mPolicy); @@ -162,40 +163,10 @@ void LLAppCoreHttp::init() continue; } } - - // Get target connection concurrency value - U32 setting(init_data[i].mDefault); - if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) - { - U32 new_setting(gSavedSettings.getU32(init_data[i].mKey)); - if (new_setting) - { - // Treat zero settings as an ask for default - setting = new_setting / init_data[i].mDivisor; - setting = llclamp(setting, init_data[i].mMin, init_data[i].mMax); - } - } - - // Set it and report - // *TODO: These are intended to be per-host limits when we can - // support that in llcorehttp/libcurl. - LLCore::HttpStatus status; - status = LLCore::HttpRequest::setPolicyClassOption(mPolicies[policy], - LLCore::HttpRequest::CP_CONNECTION_LIMIT, - setting); - if (! status) - { - LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage - << " concurrency. Reason: " << status.toString() - << LL_ENDL; - } - else if (setting != init_data[i].mDefault) - { - LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage - << " concurrency. New value: " << setting - << LL_ENDL; - } } + + // Apply initial settings + refreshSettings(true); // Kick the thread status = LLCore::HttpRequest::startThread(); @@ -206,6 +177,30 @@ void LLAppCoreHttp::init() } mRequest = new LLCore::HttpRequest; + + // Register signals for settings and state changes + for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) + { + if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) + { + LLPointer cntrl_ptr = gSavedSettings.getControl(init_data[i].mKey); + if (cntrl_ptr.isNull()) + { + LL_WARNS("Init") << "Unable to set signal on global setting '" << init_data[i].mKey + << "'" << LL_ENDL; + } + else + { + mSettingsSignal[i] = cntrl_ptr->getCommitSignal()->connect(boost::bind(&setting_changed)); + } + } + } +} + + +void setting_changed() +{ + LLAppViewer::instance()->getAppCoreHttp().refreshSettings(false); } @@ -248,6 +243,11 @@ void LLAppCoreHttp::cleanup() } } + for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) + { + mSettingsSignal[i].disconnect(); + } + delete mRequest; mRequest = NULL; @@ -260,6 +260,60 @@ void LLAppCoreHttp::cleanup() } } +void LLAppCoreHttp::refreshSettings(bool initial) +{ + for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) + { + const EAppPolicy policy(init_data[i].mPolicy); + + // Get target connection concurrency value + U32 setting(init_data[i].mDefault); + if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) + { + U32 new_setting(gSavedSettings.getU32(init_data[i].mKey)); + if (new_setting) + { + // Treat zero settings as an ask for default + setting = new_setting / init_data[i].mDivisor; + setting = llclamp(setting, init_data[i].mMin, init_data[i].mMax); + } + } + + if (! initial && setting == mSettings[policy]) + { + // Unchanged, try next setting + continue; + } + + // Set it and report + // *TODO: These are intended to be per-host limits when we can + // support that in llcorehttp/libcurl. + LLCore::HttpStatus status; + status = LLCore::HttpRequest::setPolicyClassOption(mPolicies[policy], + LLCore::HttpRequest::CP_CONNECTION_LIMIT, + setting); + if (! status) + { + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " concurrency. Reason: " << status.toString() + << LL_ENDL; + } + else + { + LL_DEBUGS("Init") << "Changed " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + mSettings[policy] = setting; + if (initial && setting != init_data[i].mDefault) + { + LL_INFOS("Init") << "Application settings overriding default " << init_data[i].mUsage + << " concurrency. New value: " << setting + << LL_ENDL; + } + } + } +} + void LLAppCoreHttp::onCompleted(LLCore::HttpHandle, LLCore::HttpResponse *) { diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 4a14c35966..6dc3bb2130 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -85,16 +85,21 @@ public: { return mPolicies[policy]; } + + // Apply initial or new settings from the environment. + void refreshSettings(bool initial); private: static const F64 MAX_THREAD_WAIT_TIME; private: - LLCore::HttpRequest * mRequest; + LLCore::HttpRequest * mRequest; // Request queue to issue shutdowns LLCore::HttpHandle mStopHandle; F64 mStopRequested; bool mStopped; - policy_t mPolicies[AP_COUNT]; + policy_t mPolicies[AP_COUNT]; // Policy class id for each connection set + U32 mSettings[AP_COUNT]; + boost::signals2::connection mSettingsSignal[AP_COUNT]; // Signals to global settings that affect us }; diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index aa10e64af8..3af9ce7342 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -259,10 +259,9 @@ const char * const LOG_MESH = "Mesh"; // Static data and functions to measure mesh load // time metrics for a new region scene. -static bool metrics_inited(false); -static boost::signals2::connection metrics_teleport_connection; static unsigned int metrics_teleport_start_count(0); -static void metrics_teleport_started(); +boost::signals2::connection metrics_teleport_started_signal; +static void teleport_started(); static bool is_retryable(LLCore::HttpStatus status); //get the number of bytes resident in memory for given volume @@ -2667,29 +2666,18 @@ void LLMeshRepository::init() apr_sleep(100); } - + metrics_teleport_started_signal = LLViewerMessage::getInstance()->setTeleportStartedCallback(teleport_started); mThread = new LLMeshRepoThread(); mThread->start(); - if (! metrics_inited) - { - // Get teleport started signals to restart timings. - metrics_teleport_connection = LLViewerMessage::getInstance()-> - setTeleportStartedCallback(metrics_teleport_started); - metrics_inited = true; - } } void LLMeshRepository::shutdown() { llinfos << "Shutting down mesh repository." << llendl; - if (metrics_inited) - { - metrics_teleport_connection.disconnect(); - metrics_inited = false; - } + metrics_teleport_started_signal.disconnect(); for (U32 i = 0; i < mUploads.size(); ++i) { @@ -4109,6 +4097,7 @@ bool LLMeshRepository::meshRezEnabled() // static void LLMeshRepository::metricsStart() { + ++metrics_teleport_start_count; sQuiescentTimer.start(0); } @@ -4127,7 +4116,6 @@ void LLMeshRepository::metricsProgress(unsigned int this_count) if (first_start) { - ++metrics_teleport_start_count; metricsStart(); first_start = false; } @@ -4157,19 +4145,13 @@ void LLMeshRepository::metricsUpdate() } } -// Will use a request to start a teleport as a signal to -// restart a timing sequence. We don't get one of these -// for login so initial start is done above. -// // Threading: main thread only // static -void metrics_teleport_started() +void teleport_started() { LLMeshRepository::metricsStart(); - ++metrics_teleport_start_count; } - // This comes from an edit in viewer-cat. Unify this once that's // available everywhere. bool is_retryable(LLCore::HttpStatus status) -- cgit v1.3 From f42d17f6c1d206a3396d9b4629246c4778ae1ca8 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Sat, 29 Jun 2013 03:08:32 +0000 Subject: Orphaned declaration preventing compilation. --- indra/newview/llappcorehttp.cpp | 1 - 1 file changed, 1 deletion(-) (limited to 'indra/newview/llappcorehttp.cpp') diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 70e349e33d..0c242e57db 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -71,7 +71,6 @@ static const struct } }; -static void teleport_started(); static void setting_changed(); -- cgit v1.3 From eff651cffca60f2b69f6c596a8e9aa9e1ab44d3c Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Fri, 12 Jul 2013 15:00:24 -0400 Subject: SH-4312 Configuration data between viewer and llcorehttp is clumsy. Much improved. Unified the global and class options into a single option list. Implemented static and dynamic setting paths as much as possible. Dynamic path does require packet/RPC but otherwise there's near unification. Dynamic modes can't get values back yet due to the response/notifier scheme but this doesn't bother me. Flatten global and class options into simpler struct-like entities. Setter/getter available on these when needed (external APIs) but code can otherwise fiddle directly when it knows what to do. Much duplicated options/state removed from HttpPolicy. Comments cleaned up. Threads better described and consistently mentioned in API docs. Integration test extended for 503 responses with Reply-After headers. --- indra/llcorehttp/_httpinternal.h | 6 +- indra/llcorehttp/_httplibcurl.h | 15 ++- indra/llcorehttp/_httpoperation.h | 8 +- indra/llcorehttp/_httpoprequest.cpp | 19 ++- indra/llcorehttp/_httpopsetget.cpp | 93 ++++++++++--- indra/llcorehttp/_httpopsetget.h | 27 ++-- indra/llcorehttp/_httppolicy.cpp | 110 ++++++++------- indra/llcorehttp/_httppolicy.h | 39 ++++-- indra/llcorehttp/_httppolicyclass.cpp | 37 ++---- indra/llcorehttp/_httppolicyclass.h | 19 ++- indra/llcorehttp/_httppolicyglobal.cpp | 66 ++++----- indra/llcorehttp/_httppolicyglobal.h | 23 +++- indra/llcorehttp/_httpservice.cpp | 170 +++++++++++++++++++++--- indra/llcorehttp/_httpservice.h | 52 +++++--- indra/llcorehttp/examples/http_texture_load.cpp | 7 +- indra/llcorehttp/httpcommon.h | 109 +++++++++++++-- indra/llcorehttp/httprequest.cpp | 107 ++++++++------- indra/llcorehttp/httprequest.h | 128 +++++++++--------- indra/llcorehttp/tests/test_httprequest.hpp | 140 ++++++++++++++++++- indra/llcorehttp/tests/test_llcorehttp_peer.py | 49 ++++++- indra/newview/llappcorehttp.cpp | 29 ++-- 21 files changed, 881 insertions(+), 372 deletions(-) (limited to 'indra/newview/llappcorehttp.cpp') diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index f085ca3b91..80f4f34942 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -36,7 +36,8 @@ // General library to-do list // // - Implement policy classes. Structure is mostly there just didn't -// need it for the first consumer. +// need it for the first consumer. [Classes are there. More +// advanced features, like borrowing, aren't there yet.] // - Consider Removing 'priority' from the request interface. Its use // in an always active class can lead to starvation of low-priority // requests. Requires coodination of priority values across all @@ -46,6 +47,7 @@ // may not really need it. // - Set/get for global policy and policy classes is clumsy. Rework // it heading in a direction that allows for more dynamic behavior. +// [Mostly fixed] // - Move HttpOpRequest::prepareRequest() to HttpLibcurl for the // pedantic. // - Update downloader and other long-duration services are going to @@ -73,7 +75,7 @@ // the main source file. // - Expand areas of usage eventually leading to the removal of LLCurl. // Rough order of expansion: -// . Mesh fetch +// . Mesh fetch [Underway] // . Avatar names // . Group membership lists // . Caps access in general diff --git a/indra/llcorehttp/_httplibcurl.h b/indra/llcorehttp/_httplibcurl.h index 611f029ef5..0ec90437bb 100755 --- a/indra/llcorehttp/_httplibcurl.h +++ b/indra/llcorehttp/_httplibcurl.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -71,16 +71,22 @@ public: /// /// @return Indication of how long this method is /// willing to wait for next service call. + /// + /// Threading: called by worker thread. HttpService::ELoopSpeed processTransport(); /// Add request to the active list. Caller is expected to have /// provided us with a reference count on the op to hold the /// request. (No additional references will be added.) + /// + /// Threading: called by worker thread. void addOp(HttpOpRequest * op); /// One-time call to set the number of policy classes to be /// serviced and to create the resources for each. Value /// must agree with HttpPolicy::setPolicies() call. + /// + /// Threading: called by init thread. void start(int policy_count); /// Synchronously stop libcurl operations. All active requests @@ -91,9 +97,13 @@ public: /// respective reply queues. /// /// Can be restarted with a start() call. + /// + /// Threading: called by worker thread. void shutdown(); /// Return global and per-class counts of active requests. + /// + /// Threading: called by worker thread. int getActiveCount() const; int getActiveCountInClass(int policy_class) const; @@ -103,6 +113,7 @@ public: /// /// @return True if handle was found and operation canceled. /// + /// Threading: called by worker thread. bool cancel(HttpHandle handle); protected: @@ -121,7 +132,7 @@ protected: HttpService * mService; // Simple reference, not owner active_set_t mActiveOps; int mPolicyCount; - CURLM ** mMultiHandles; + CURLM ** mMultiHandles; // One handle per policy class }; // end class HttpLibcurl } // end namespace LLCore diff --git a/indra/llcorehttp/_httpoperation.h b/indra/llcorehttp/_httpoperation.h index 914627fad0..937a61187d 100755 --- a/indra/llcorehttp/_httpoperation.h +++ b/indra/llcorehttp/_httpoperation.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -72,7 +72,7 @@ class HttpService; class HttpOperation : public LLCoreInt::RefCounted { public: - /// Threading: called by a consumer/application thread. + /// Threading: called by consumer thread. HttpOperation(); protected: @@ -108,7 +108,7 @@ public: /// by the worker thread. This is passible data /// until notification is performed. /// - /// Threading: called by application thread. + /// Threading: called by consumer thread. /// void setReplyPath(HttpReplyQueue * reply_queue, HttpHandler * handler); @@ -141,7 +141,7 @@ public: /// call to HttpRequest::update(). This method does the necessary /// dispatching. /// - /// Threading: called by application thread. + /// Threading: called by consumer thread. /// virtual void visitNotifier(HttpRequest *); diff --git a/indra/llcorehttp/_httpoprequest.cpp b/indra/llcorehttp/_httpoprequest.cpp index ce0fc605ab..d72f8f6119 100755 --- a/indra/llcorehttp/_httpoprequest.cpp +++ b/indra/llcorehttp/_httpoprequest.cpp @@ -402,7 +402,7 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) // *FIXME: better error handling later HttpStatus status; - // Get policy options + // Get global policy options HttpPolicyGlobal & policy(service->getPolicy().getGlobalOptions()); mCurlHandle = curl_easy_init(); @@ -441,30 +441,27 @@ HttpStatus HttpOpRequest::prepareRequest(HttpService * service) curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYPEER, 1); curl_easy_setopt(mCurlHandle, CURLOPT_SSL_VERIFYHOST, 0); - const std::string * opt_value(NULL); - long opt_long(0L); - policy.get(HttpRequest::GP_LLPROXY, &opt_long); - if (opt_long) + if (policy.mUseLLProxy) { // Use the viewer-based thread-safe API which has a // fast/safe check for proxy enable. Would like to // encapsulate this someway... LLProxy::getInstance()->applyProxySettings(mCurlHandle); } - else if (policy.get(HttpRequest::GP_HTTP_PROXY, &opt_value)) + else if (policy.mHttpProxy.size()) { // *TODO: This is fine for now but get fuller socks5/ // authentication thing going later.... - curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, opt_value->c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, policy.mHttpProxy.c_str()); curl_easy_setopt(mCurlHandle, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); } - if (policy.get(HttpRequest::GP_CA_PATH, &opt_value)) + if (policy.mCAPath.size()) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, opt_value->c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_CAPATH, policy.mCAPath.c_str()); } - if (policy.get(HttpRequest::GP_CA_FILE, &opt_value)) + if (policy.mCAFile.size()) { - curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, opt_value->c_str()); + curl_easy_setopt(mCurlHandle, CURLOPT_CAINFO, policy.mCAFile.c_str()); } switch (mReqMethod) diff --git a/indra/llcorehttp/_httpopsetget.cpp b/indra/llcorehttp/_httpopsetget.cpp index 8198528a9b..a5363f9170 100755 --- a/indra/llcorehttp/_httpopsetget.cpp +++ b/indra/llcorehttp/_httpopsetget.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -27,6 +27,7 @@ #include "_httpopsetget.h" #include "httpcommon.h" +#include "httprequest.h" #include "_httpservice.h" #include "_httppolicy.h" @@ -43,10 +44,11 @@ namespace LLCore HttpOpSetGet::HttpOpSetGet() : HttpOperation(), - mIsGlobal(false), - mDoSet(false), - mSetting(-1), // Nothing requested - mLongValue(0L) + mReqOption(HttpRequest::PO_CONNECTION_LIMIT), + mReqClass(HttpRequest::INVALID_POLICY_ID), + mReqDoSet(false), + mReqLongValue(0L), + mReplyLongValue(0L) {} @@ -54,37 +56,84 @@ HttpOpSetGet::~HttpOpSetGet() {} -void HttpOpSetGet::setupGet(HttpRequest::EGlobalPolicy setting) +HttpStatus HttpOpSetGet::setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass) { - mIsGlobal = true; - mSetting = setting; + HttpStatus status; + + mReqOption = opt; + mReqClass = pclass; + return status; } -void HttpOpSetGet::setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value) +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value) { - mIsGlobal = true; - mDoSet = true; - mSetting = setting; - mStrValue = value; + HttpStatus status; + + if (! HttpService::sOptionDesc[opt].mIsLong) + { + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + if (! HttpService::sOptionDesc[opt].mIsDynamic) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + + mReqOption = opt; + mReqClass = pclass; + mReqDoSet = true; + mReqLongValue = value; + + return status; } -void HttpOpSetGet::stageFromRequest(HttpService * service) +HttpStatus HttpOpSetGet::setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value) { - HttpPolicyGlobal & pol_opt(service->getPolicy().getGlobalOptions()); - HttpRequest::EGlobalPolicy setting(static_cast(mSetting)); + HttpStatus status; + + if (HttpService::sOptionDesc[opt].mIsLong) + { + return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); + } + if (! HttpService::sOptionDesc[opt].mIsDynamic) + { + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + } + + mReqOption = opt; + mReqClass = pclass; + mReqDoSet = true; + mReqStrValue = value; - if (mDoSet) + return status; +} + + +void HttpOpSetGet::stageFromRequest(HttpService * service) +{ + if (mReqDoSet) { - mStatus = pol_opt.set(setting, mStrValue); + if (HttpService::sOptionDesc[mReqOption].mIsLong) + { + mStatus = service->setPolicyOption(mReqOption, mReqClass, + mReqLongValue, &mReplyLongValue); + } + else + { + mStatus = service->setPolicyOption(mReqOption, mReqClass, + mReqStrValue, &mReplyStrValue); + } } - if (mStatus) + else { - const std::string * value(NULL); - if ((mStatus = pol_opt.get(setting, &value))) + if (HttpService::sOptionDesc[mReqOption].mIsLong) + { + mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyLongValue); + } + else { - mStrValue = *value; + mStatus = service->getPolicyOption(mReqOption, mReqClass, &mReplyStrValue); } } diff --git a/indra/llcorehttp/_httpopsetget.h b/indra/llcorehttp/_httpopsetget.h index 6966b9d94e..a1e76dd429 100755 --- a/indra/llcorehttp/_httpopsetget.h +++ b/indra/llcorehttp/_httpopsetget.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -46,7 +46,10 @@ namespace LLCore /// configuration settings. /// /// *NOTE: Expect this to change. Don't really like it yet. - +/// +/// *TODO: Can't return values to caller yet. Need to do +/// something better with HttpResponse and visitNotifier(). +/// class HttpOpSetGet : public HttpOperation { public: @@ -61,19 +64,23 @@ private: public: /// Threading: called by application thread - void setupGet(HttpRequest::EGlobalPolicy setting); - void setupSet(HttpRequest::EGlobalPolicy setting, const std::string & value); + HttpStatus setupGet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass); + HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, long value); + HttpStatus setupSet(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, const std::string & value); virtual void stageFromRequest(HttpService *); public: // Request data - bool mIsGlobal; - bool mDoSet; - int mSetting; - long mLongValue; - std::string mStrValue; - + HttpRequest::EPolicyOption mReqOption; + HttpRequest::policy_t mReqClass; + bool mReqDoSet; + long mReqLongValue; + std::string mReqStrValue; + + // Reply Data + long mReplyLongValue; + std::string mReplyStrValue; }; // end class HttpOpSetGet diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 5f303dd0fe..2754e8ef07 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -41,57 +41,64 @@ namespace LLCore // Per-policy-class data for a running system. -// Collection of queues, parameters, history, metrics, etc. +// Collection of queues, options and other data // for a single policy class. // // Threading: accessed only by worker thread -struct HttpPolicy::State +struct HttpPolicy::ClassState { public: - State() - : mConnMax(HTTP_CONNECTION_LIMIT_DEFAULT), - mConnAt(HTTP_CONNECTION_LIMIT_DEFAULT), - mConnMin(1), - mNextSample(0), - mErrorCount(0), - mErrorFactor(0) + ClassState() {} HttpReadyQueue mReadyQueue; HttpRetryQueue mRetryQueue; HttpPolicyClass mOptions; - - long mConnMax; - long mConnAt; - long mConnMin; - - HttpTime mNextSample; - unsigned long mErrorCount; - unsigned long mErrorFactor; }; HttpPolicy::HttpPolicy(HttpService * service) - : mActiveClasses(0), - mState(NULL), - mService(service) -{} + : mService(service) +{ + // Create default class + mClasses.push_back(new ClassState()); +} HttpPolicy::~HttpPolicy() { shutdown(); + + for (class_list_t::iterator it(mClasses.begin()); it != mClasses.end(); ++it) + { + delete (*it); + } + mClasses.clear(); mService = NULL; } +HttpRequest::policy_t HttpPolicy::createPolicyClass() +{ + const HttpRequest::policy_t policy_class(mClasses.size()); + if (policy_class >= HTTP_POLICY_CLASS_LIMIT) + { + return HttpRequest::INVALID_POLICY_ID; + } + mClasses.push_back(new ClassState()); + return policy_class; +} + + void HttpPolicy::shutdown() { - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - HttpRetryQueue & retryq(mState[policy_class].mRetryQueue); + ClassState & state(*mClasses[policy_class]); + + HttpRetryQueue & retryq(state.mRetryQueue); while (! retryq.empty()) { HttpOpRequest * op(retryq.top()); @@ -101,7 +108,7 @@ void HttpPolicy::shutdown() op->release(); } - HttpReadyQueue & readyq(mState[policy_class].mReadyQueue); + HttpReadyQueue & readyq(state.mReadyQueue); while (! readyq.empty()) { HttpOpRequest * op(readyq.top()); @@ -111,28 +118,11 @@ void HttpPolicy::shutdown() op->release(); } } - delete [] mState; - mState = NULL; - mActiveClasses = 0; } -void HttpPolicy::start(const HttpPolicyGlobal & global, - const std::vector & classes) -{ - llassert_always(! mState); - - mGlobalOptions = global; - mActiveClasses = classes.size(); - mState = new State [mActiveClasses]; - for (int i(0); i < mActiveClasses; ++i) - { - mState[i].mOptions = classes[i]; - mState[i].mConnMax = classes[i].mConnectionLimit; - mState[i].mConnAt = mState[i].mConnMax; - mState[i].mConnMin = 2; - } -} +void HttpPolicy::start() +{} void HttpPolicy::addOp(HttpOpRequest * op) @@ -141,7 +131,7 @@ void HttpPolicy::addOp(HttpOpRequest * op) op->mPolicyRetries = 0; op->mPolicy503Retries = 0; - mState[policy_class].mReadyQueue.push(op); + mClasses[policy_class]->mReadyQueue.push(op); } @@ -183,7 +173,7 @@ void HttpPolicy::retryOp(HttpOpRequest * op) << static_cast(op) << LL_ENDL; } - mState[policy_class].mRetryQueue.push(op); + mClasses[policy_class]->mRetryQueue.push(op); } @@ -204,11 +194,11 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() HttpService::ELoopSpeed result(HttpService::REQUEST_SLEEP); HttpLibcurl & transport(mService->getTransport()); - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - State & state(mState[policy_class]); + ClassState & state(*mClasses[policy_class]); int active(transport.getActiveCountInClass(policy_class)); - int needed(state.mConnAt - active); // Expect negatives here + int needed(state.mOptions.mConnectionLimit - active); // Expect negatives here HttpRetryQueue & retryq(state.mRetryQueue); HttpReadyQueue & readyq(state.mReadyQueue); @@ -256,9 +246,9 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t priority) { - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - State & state(mState[policy_class]); + ClassState & state(*mClasses[policy_class]); // We don't scan retry queue because a priority change there // is meaningless. The request will be issued based on retry // intervals not priority value, which is now moot. @@ -286,9 +276,9 @@ bool HttpPolicy::changePriority(HttpHandle handle, HttpRequest::priority_t prior bool HttpPolicy::cancel(HttpHandle handle) { - for (int policy_class(0); policy_class < mActiveClasses; ++policy_class) + for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { - State & state(mState[policy_class]); + ClassState & state(*mClasses[policy_class]); // Scan retry queue HttpRetryQueue::container_type & c1(state.mRetryQueue.get_container()); @@ -382,13 +372,21 @@ bool HttpPolicy::stageAfterCompletion(HttpOpRequest * op) return false; // not active } + +HttpPolicyClass & HttpPolicy::getClassOptions(HttpRequest::policy_t pclass) +{ + llassert_always(pclass >= 0 && pclass < mClasses.size()); + + return mClasses[pclass]->mOptions; +} + int HttpPolicy::getReadyCount(HttpRequest::policy_t policy_class) const { - if (policy_class < mActiveClasses) + if (policy_class < mClasses.size()) { - return (mState[policy_class].mReadyQueue.size() - + mState[policy_class].mRetryQueue.size()); + return (mClasses[policy_class]->mReadyQueue.size() + + mClasses[policy_class]->mRetryQueue.size()); } return 0; } diff --git a/indra/llcorehttp/_httppolicy.h b/indra/llcorehttp/_httppolicy.h index 03d92c0b8e..bf1aa74267 100755 --- a/indra/llcorehttp/_httppolicy.h +++ b/indra/llcorehttp/_httppolicy.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -60,6 +60,9 @@ private: void operator=(const HttpPolicy &); // Not defined public: + /// Threading: called by init thread. + HttpRequest::policy_t createPolicyClass(); + /// Cancel all ready and retry requests sending them to /// their notification queues. Release state resources /// making further request handling impossible. @@ -71,9 +74,8 @@ public: /// requests. One-time call invoked before starting /// the worker thread. /// - /// Threading: called by application thread - void start(const HttpPolicyGlobal & global, - const std::vector & classes); + /// Threading: called by init thread + void start(); /// Give the policy layer some cycles to scan the ready /// queue promoting higher-priority requests to active @@ -93,7 +95,7 @@ public: /// and should not be modified by anyone until retrieved /// from queue. /// - /// Threading: called by any thread + /// Threading: called by worker thread void addOp(HttpOpRequest *); /// Similar to addOp, used when a caller wants to retry a @@ -130,30 +132,39 @@ public: /// Threading: called by worker thread bool stageAfterCompletion(HttpOpRequest * op); - // Get pointer to global policy options. Caller is expected - // to do context checks like no setting once running. + /// Get a reference to global policy options. Caller is expected + /// to do context checks like no setting once running. These + /// are done, for example, in @see HttpService interfaces. /// /// Threading: called by any thread *but* the object may /// only be modified by the worker thread once running. - /// HttpPolicyGlobal & getGlobalOptions() { return mGlobalOptions; } + /// Get a reference to class policy options. Caller is expected + /// to do context checks like no setting once running. These + /// are done, for example, in @see HttpService interfaces. + /// + /// Threading: called by any thread *but* the object may + /// only be modified by the worker thread once running and + /// read accesses by other threads are exposed to races at + /// that point. + HttpPolicyClass & getClassOptions(HttpRequest::policy_t pclass); + /// Get ready counts for a particular policy class /// /// Threading: called by worker thread int getReadyCount(HttpRequest::policy_t policy_class) const; protected: - struct State; - - int mActiveClasses; - State * mState; - HttpService * mService; // Naked pointer, not refcounted, not owner - HttpPolicyGlobal mGlobalOptions; + struct ClassState; + typedef std::vector class_list_t; + HttpPolicyGlobal mGlobalOptions; + class_list_t mClasses; + HttpService * mService; // Naked pointer, not refcounted, not owner }; // end class HttpPolicy } // end namespace LLCore diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp index 1a55ab1ac6..fe4359081a 100755 --- a/indra/llcorehttp/_httppolicyclass.cpp +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -34,8 +34,7 @@ namespace LLCore HttpPolicyClass::HttpPolicyClass() - : mSetMask(0UL), - mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mPipelining(HTTP_PIPELINING_DEFAULT) {} @@ -49,7 +48,6 @@ HttpPolicyClass & HttpPolicyClass::operator=(const HttpPolicyClass & other) { if (this != &other) { - mSetMask = other.mSetMask; mConnectionLimit = other.mConnectionLimit; mPerHostConnectionLimit = other.mPerHostConnectionLimit; mPipelining = other.mPipelining; @@ -59,26 +57,25 @@ HttpPolicyClass & HttpPolicyClass::operator=(const HttpPolicyClass & other) HttpPolicyClass::HttpPolicyClass(const HttpPolicyClass & other) - : mSetMask(other.mSetMask), - mConnectionLimit(other.mConnectionLimit), + : mConnectionLimit(other.mConnectionLimit), mPerHostConnectionLimit(other.mPerHostConnectionLimit), mPipelining(other.mPipelining) {} -HttpStatus HttpPolicyClass::set(HttpRequest::EClassPolicy opt, long value) +HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value) { switch (opt) { - case HttpRequest::CP_CONNECTION_LIMIT: + case HttpRequest::PO_CONNECTION_LIMIT: mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); break; - case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: + case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: mPerHostConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), mConnectionLimit); break; - case HttpRequest::CP_ENABLE_PIPELINING: + case HttpRequest::PO_ENABLE_PIPELINING: mPipelining = llclamp(value, 0L, 1L); break; @@ -86,38 +83,30 @@ HttpStatus HttpPolicyClass::set(HttpRequest::EClassPolicy opt, long value) return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mSetMask |= 1UL << int(opt); return HttpStatus(); } -HttpStatus HttpPolicyClass::get(HttpRequest::EClassPolicy opt, long * value) +HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) const { - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - long * src(NULL); - switch (opt) { - case HttpRequest::CP_CONNECTION_LIMIT: - src = &mConnectionLimit; + case HttpRequest::PO_CONNECTION_LIMIT: + *value = mConnectionLimit; break; - case HttpRequest::CP_PER_HOST_CONNECTION_LIMIT: - src = &mPerHostConnectionLimit; + case HttpRequest::PO_PER_HOST_CONNECTION_LIMIT: + *value = mPerHostConnectionLimit; break; - case HttpRequest::CP_ENABLE_PIPELINING: - src = &mPipelining; + case HttpRequest::PO_ENABLE_PIPELINING: + *value = mPipelining; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - - *value = *src; return HttpStatus(); } diff --git a/indra/llcorehttp/_httppolicyclass.h b/indra/llcorehttp/_httppolicyclass.h index d175413cbd..69fb459d22 100755 --- a/indra/llcorehttp/_httppolicyclass.h +++ b/indra/llcorehttp/_httppolicyclass.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,6 +34,18 @@ namespace LLCore { +/// Options struct for per-class policy options. +/// +/// Combines both raw blob data access with semantics-enforcing +/// set/get interfaces. For internal operations by the worker +/// thread, just grab the setting directly from instance and test/use +/// as needed. When attached to external APIs (the public API +/// options interfaces) the set/get methods are available to +/// enforce correct ranges, data types, contexts, etc. and suitable +/// status values are returned. +/// +/// Threading: Single-threaded. In practice, init thread before +/// worker starts, worker thread after. class HttpPolicyClass { public: @@ -44,11 +56,10 @@ public: HttpPolicyClass(const HttpPolicyClass &); // Not defined public: - HttpStatus set(HttpRequest::EClassPolicy opt, long value); - HttpStatus get(HttpRequest::EClassPolicy opt, long * value); + HttpStatus set(HttpRequest::EPolicyOption opt, long value); + HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; public: - unsigned long mSetMask; long mConnectionLimit; long mPerHostConnectionLimit; long mPipelining; diff --git a/indra/llcorehttp/_httppolicyglobal.cpp b/indra/llcorehttp/_httppolicyglobal.cpp index 72f409d3b1..1dc95f3dce 100755 --- a/indra/llcorehttp/_httppolicyglobal.cpp +++ b/indra/llcorehttp/_httppolicyglobal.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,8 +34,7 @@ namespace LLCore HttpPolicyGlobal::HttpPolicyGlobal() - : mSetMask(0UL), - mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), + : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mTrace(HTTP_TRACE_OFF), mUseLLProxy(0) {} @@ -49,7 +48,6 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) { if (this != &other) { - mSetMask = other.mSetMask; mConnectionLimit = other.mConnectionLimit; mCAPath = other.mCAPath; mCAFile = other.mCAFile; @@ -61,19 +59,19 @@ HttpPolicyGlobal & HttpPolicyGlobal::operator=(const HttpPolicyGlobal & other) } -HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, long value) { switch (opt) { - case HttpRequest::GP_CONNECTION_LIMIT: + case HttpRequest::PO_CONNECTION_LIMIT: mConnectionLimit = llclamp(value, long(HTTP_CONNECTION_LIMIT_MIN), long(HTTP_CONNECTION_LIMIT_MAX)); break; - case HttpRequest::GP_TRACE: + case HttpRequest::PO_TRACE: mTrace = llclamp(value, long(HTTP_TRACE_MIN), long(HTTP_TRACE_MAX)); break; - case HttpRequest::GP_LLPROXY: + case HttpRequest::PO_LLPROXY: mUseLLProxy = llclamp(value, 0L, 1L); break; @@ -81,24 +79,23 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, long value) return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mSetMask |= 1UL << int(opt); return HttpStatus(); } -HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::string & value) +HttpStatus HttpPolicyGlobal::set(HttpRequest::EPolicyOption opt, const std::string & value) { switch (opt) { - case HttpRequest::GP_CA_PATH: + case HttpRequest::PO_CA_PATH: mCAPath = value; break; - case HttpRequest::GP_CA_FILE: + case HttpRequest::PO_CA_FILE: mCAFile = value; break; - case HttpRequest::GP_HTTP_PROXY: + case HttpRequest::PO_HTTP_PROXY: mCAFile = value; break; @@ -106,69 +103,54 @@ HttpStatus HttpPolicyGlobal::set(HttpRequest::EGlobalPolicy opt, const std::stri return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - mSetMask |= 1UL << int(opt); return HttpStatus(); } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, long * value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, long * value) const { - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - long * src(NULL); - switch (opt) { - case HttpRequest::GP_CONNECTION_LIMIT: - src = &mConnectionLimit; + case HttpRequest::PO_CONNECTION_LIMIT: + *value = mConnectionLimit; break; - case HttpRequest::GP_TRACE: - src = &mTrace; + case HttpRequest::PO_TRACE: + *value = mTrace; break; - case HttpRequest::GP_LLPROXY: - src = &mUseLLProxy; + case HttpRequest::PO_LLPROXY: + *value = mUseLLProxy; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - - *value = *src; return HttpStatus(); } -HttpStatus HttpPolicyGlobal::get(HttpRequest::EGlobalPolicy opt, const std::string ** value) +HttpStatus HttpPolicyGlobal::get(HttpRequest::EPolicyOption opt, std::string * value) const { - static const HttpStatus not_set(HttpStatus::LLCORE, HE_OPT_NOT_SET); - const std::string * src(NULL); - switch (opt) { - case HttpRequest::GP_CA_PATH: - src = &mCAPath; + case HttpRequest::PO_CA_PATH: + *value = mCAPath; break; - case HttpRequest::GP_CA_FILE: - src = &mCAFile; + case HttpRequest::PO_CA_FILE: + *value = mCAFile; break; - case HttpRequest::GP_HTTP_PROXY: - src = &mHttpProxy; + case HttpRequest::PO_HTTP_PROXY: + *value = mHttpProxy; break; default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } - if (! (mSetMask & (1UL << int(opt)))) - return not_set; - - *value = src; return HttpStatus(); } diff --git a/indra/llcorehttp/_httppolicyglobal.h b/indra/llcorehttp/_httppolicyglobal.h index a50d0e4188..67c4ba9481 100755 --- a/indra/llcorehttp/_httppolicyglobal.h +++ b/indra/llcorehttp/_httppolicyglobal.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -34,6 +34,18 @@ namespace LLCore { +/// Options struct for global policy options. +/// +/// Combines both raw blob data access with semantics-enforcing +/// set/get interfaces. For internal operations by the worker +/// thread, just grab the setting directly from instance and test/use +/// as needed. When attached to external APIs (the public API +/// options interfaces) the set/get methods are available to +/// enforce correct ranges, data types, contexts, etc. and suitable +/// status values are returned. +/// +/// Threading: Single-threaded. In practice, init thread before +/// worker starts, worker thread after. class HttpPolicyGlobal { public: @@ -46,13 +58,12 @@ private: HttpPolicyGlobal(const HttpPolicyGlobal &); // Not defined public: - HttpStatus set(HttpRequest::EGlobalPolicy opt, long value); - HttpStatus set(HttpRequest::EGlobalPolicy opt, const std::string & value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, long * value); - HttpStatus get(HttpRequest::EGlobalPolicy opt, const std::string ** value); + HttpStatus set(HttpRequest::EPolicyOption opt, long value); + HttpStatus set(HttpRequest::EPolicyOption opt, const std::string & value); + HttpStatus get(HttpRequest::EPolicyOption opt, long * value) const; + HttpStatus get(HttpRequest::EPolicyOption opt, std::string * value) const; public: - unsigned long mSetMask; long mConnectionLimit; std::string mCAPath; std::string mCAFile; diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index 0821401289..e21d196a3e 100755 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -43,6 +43,17 @@ namespace LLCore { +const HttpService::OptionDescriptor HttpService::sOptionDesc[] = +{ // isLong isDynamic isGlobal isClass + { true, true, true, true }, // PO_CONNECTION_LIMIT + { true, true, false, true }, // PO_PER_HOST_CONNECTION_LIMIT + { false, false, true, false }, // PO_CA_PATH + { false, false, true, false }, // PO_CA_FILE + { false, true, true, false }, // PO_HTTP_PROXY + { true, true, true, false }, // PO_LLPROXY + { true, true, true, false }, // PO_TRACE + { true, true, false, true } // PO_ENABLE_PIPELINING +}; HttpService * HttpService::sInstance(NULL); volatile HttpService::EState HttpService::sState(NOT_INITIALIZED); @@ -51,12 +62,9 @@ HttpService::HttpService() mExitRequested(0U), mThread(NULL), mPolicy(NULL), - mTransport(NULL) -{ - // Create the default policy class - HttpPolicyClass pol_class; - mPolicyClasses.push_back(pol_class); -} + mTransport(NULL), + mLastPolicy(0) +{} HttpService::~HttpService() @@ -146,13 +154,8 @@ void HttpService::term() HttpRequest::policy_t HttpService::createPolicyClass() { - const HttpRequest::policy_t policy_class(mPolicyClasses.size()); - if (policy_class >= HTTP_POLICY_CLASS_LIMIT) - { - return 0; - } - mPolicyClasses.push_back(HttpPolicyClass()); - return policy_class; + mLastPolicy = mPolicy->createPolicyClass(); + return mLastPolicy; } @@ -185,8 +188,8 @@ void HttpService::startThread() } // Push current policy definitions, enable policy & transport components - mPolicy->start(mPolicyGlobal, mPolicyClasses); - mTransport->start(mPolicyClasses.size()); + mPolicy->start(); + mTransport->start(mLastPolicy + 1); mThread = new LLCoreInt::HttpThread(boost::bind(&HttpService::threadRun, this, _1)); sState = RUNNING; @@ -319,7 +322,7 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) { // Setup for subsequent tracing long tracing(HTTP_TRACE_OFF); - mPolicy->getGlobalOptions().get(HttpRequest::GP_TRACE, &tracing); + mPolicy->getGlobalOptions().get(HttpRequest::PO_TRACE, &tracing); op->mTracing = (std::max)(op->mTracing, int(tracing)); if (op->mTracing > HTTP_TRACE_OFF) @@ -342,4 +345,137 @@ HttpService::ELoopSpeed HttpService::processRequestQueue(ELoopSpeed loop) } +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + long * ret_value) +{ + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (! sOptionDesc[opt].mIsLong) // datatype is long + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted + // can always get, no dynamic check + { + return HttpStatus(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + } + + HttpStatus status; + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.get(opt, ret_value); + } + else + { + HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); + + status = opts.get(opt, ret_value); + } + + return status; +} + + +HttpStatus HttpService::getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + std::string * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (sOptionDesc[opt].mIsLong) // datatype is string + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass)) // class setting permitted + // can always get, no dynamic check + { + return status; + } + + // Only global has string values + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.get(opt, ret_value); + } + + return status; +} + + +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + long value, long * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (! sOptionDesc[opt].mIsLong) // datatype is long + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted + || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted + { + return status; + } + + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + else + { + HttpPolicyClass & opts(mPolicy->getClassOptions(pclass)); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + + return status; +} + + +HttpStatus HttpService::setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t pclass, + const std::string & value, std::string * ret_value) +{ + HttpStatus status(HttpStatus::LLCORE, LLCore::HE_INVALID_ARG); + + if (opt < HttpRequest::PO_CONNECTION_LIMIT // option must be in range + || opt >= HttpRequest::PO_LAST // ditto + || (sOptionDesc[opt].mIsLong) // datatype is string + || (pclass != HttpRequest::GLOBAL_POLICY_ID && pclass > mLastPolicy) // pclass in valid range + || (pclass == HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsGlobal) // global setting permitted + || (pclass != HttpRequest::GLOBAL_POLICY_ID && ! sOptionDesc[opt].mIsClass) // class setting permitted + || (RUNNING == sState && ! sOptionDesc[opt].mIsDynamic)) // dynamic setting permitted + { + return status; + } + + // Only string values are global at this time + if (pclass == HttpRequest::GLOBAL_POLICY_ID) + { + HttpPolicyGlobal & opts(mPolicy->getGlobalOptions()); + + status = opts.set(opt, value); + if (status && ret_value) + { + status = opts.get(opt, ret_value); + } + } + + return status; +} + + } // end namespace LLCore diff --git a/indra/llcorehttp/_httpservice.h b/indra/llcorehttp/_httpservice.h index ffe0349d4d..cf23f3ab61 100755 --- a/indra/llcorehttp/_httpservice.h +++ b/indra/llcorehttp/_httpservice.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -53,6 +53,7 @@ namespace LLCore class HttpRequestQueue; class HttpPolicy; class HttpLibcurl; +class HttpOpSetGet; /// The HttpService class does the work behind the request queue. It @@ -106,7 +107,7 @@ public: NORMAL, ///< continuous polling of request, ready, active queues REQUEST_SLEEP ///< can sleep indefinitely waiting for request queue write }; - + static void init(HttpRequestQueue *); static void term(); @@ -136,7 +137,7 @@ public: /// acquires its weaknesses. static bool isStopped(); - /// Threading: callable by consumer thread *once*. + /// Threading: callable by init thread *once*. void startThread(); /// Threading: callable by worker thread. @@ -180,28 +181,39 @@ public: return *mRequestQueue; } - /// Threading: callable by consumer thread. - HttpPolicyGlobal & getGlobalOptions() - { - return mPolicyGlobal; - } - /// Threading: callable by consumer thread. HttpRequest::policy_t createPolicyClass(); - /// Threading: callable by consumer thread. - HttpPolicyClass & getClassOptions(HttpRequest::policy_t policy_class) - { - llassert(policy_class >= 0 && policy_class < mPolicyClasses.size()); - return mPolicyClasses[policy_class]; - } - protected: void threadRun(LLCoreInt::HttpThread * thread); ELoopSpeed processRequestQueue(ELoopSpeed loop); + +protected: + friend class HttpOpSetGet; + friend class HttpRequest; + + // Used internally to describe what operations are allowed + // on each policy option. + struct OptionDescriptor + { + bool mIsLong; + bool mIsDynamic; + bool mIsGlobal; + bool mIsClass; + }; + + HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + long * ret_value); + HttpStatus getPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + std::string * ret_value); + HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + long value, long * ret_value); + HttpStatus setPolicyOption(HttpRequest::EPolicyOption opt, HttpRequest::policy_t, + const std::string & value, std::string * ret_value); protected: + static const OptionDescriptor sOptionDesc[HttpRequest::PO_LAST]; static HttpService * sInstance; // === shared data === @@ -210,13 +222,13 @@ protected: LLAtomicU32 mExitRequested; LLCoreInt::HttpThread * mThread; - // === consumer-thread-only data === - HttpPolicyGlobal mPolicyGlobal; - std::vector mPolicyClasses; - // === working-thread-only data === HttpPolicy * mPolicy; // Simple pointer, has ownership HttpLibcurl * mTransport; // Simple pointer, has ownership + + // === main-thread-only data === + HttpRequest::policy_t mLastPolicy; + }; // end class HttpService } // end namespace LLCore diff --git a/indra/llcorehttp/examples/http_texture_load.cpp b/indra/llcorehttp/examples/http_texture_load.cpp index 88c8102b27..73c49687d7 100755 --- a/indra/llcorehttp/examples/http_texture_load.cpp +++ b/indra/llcorehttp/examples/http_texture_load.cpp @@ -236,9 +236,10 @@ int main(int argc, char** argv) // Initialization init_curl(); LLCore::HttpRequest::createService(); - LLCore::HttpRequest::setPolicyClassOption(LLCore::HttpRequest::DEFAULT_POLICY_ID, - LLCore::HttpRequest::CP_CONNECTION_LIMIT, - concurrency_limit); + LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, + LLCore::HttpRequest::DEFAULT_POLICY_ID, + concurrency_limit, + NULL); LLCore::HttpRequest::startThread(); // Get service point diff --git a/indra/llcorehttp/httpcommon.h b/indra/llcorehttp/httpcommon.h index c0d4ec5aad..9db884057f 100755 --- a/indra/llcorehttp/httpcommon.h +++ b/indra/llcorehttp/httpcommon.h @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -29,9 +29,9 @@ /// @package LLCore::HTTP /// -/// This library implements a high-level, Indra-code-free client interface to -/// HTTP services based on actual patterns found in the viewer and simulator. -/// Interfaces are similar to those supplied by the legacy classes +/// This library implements a high-level, Indra-code-free (somewhat) client +/// interface to HTTP services based on actual patterns found in the viewer +/// and simulator. Interfaces are similar to those supplied by the legacy classes /// LLCurlRequest and LLHTTPClient. To that is added a policy scheme that /// allows an application to specify connection behaviors: limits on /// connections, HTTP keepalive, HTTP pipelining, retry-on-error limits, etc. @@ -52,7 +52,7 @@ /// - "llcorehttp/httprequest.h" /// - "llcorehttp/httpresponse.h" /// -/// The library is still under early development and particular users +/// The library is still under development and particular users /// may need access to internal implementation details that are found /// in the _*.h header files. But this is a crutch to be avoided if at /// all possible and probably indicates some interface work is neeeded. @@ -66,6 +66,8 @@ /// . CRYPTO_set_id_callback(...) /// - HttpRequest::createService() called to instantiate singletons /// and support objects. +/// - HttpRequest::startThread() to kick off the worker thread and +/// begin servicing requests. /// /// An HTTP consumer in an application, and an application may have many /// consumers, does a few things: @@ -91,10 +93,12 @@ /// objects. /// - Do completion processing in your onCompletion() method. /// -/// Code fragments: -/// Rather than a poorly-maintained example in comments, look in the -/// example subdirectory which is a minimal yet functional tool to do -/// GET request performance testing. With four calls: +/// Code fragments. +/// +/// Initialization. Rather than a poorly-maintained example in +/// comments, look in the example subdirectory which is a minimal +/// yet functional tool to do GET request performance testing. +/// With four calls: /// /// init_curl(); /// LLCore::HttpRequest::createService(); @@ -103,7 +107,85 @@ /// /// the program is basically ready to issue requests. /// - +/// HttpHandler. Having started life as a non-indra library, +/// this code broke away from the classic Responder model and +/// introduced a handler class to represent an interface for +/// request responses. This is a non-reference-counted entity +/// which can be used as a base class or a mixin. An instance +/// of a handler can be used for each request or can be shared +/// among any number of requests. Your choice but expect to +/// code something like the following: +/// +/// class AppHandler : public LLCore::HttpHandler +/// { +/// public: +/// virtual void onCompleted(HttpHandle handle, +/// HttpResponse * response) +/// { +/// ... +/// } +/// ... +/// }; +/// ... +/// handler = new handler(...); +/// +/// +/// Issuing requests. Using 'hr' above, +/// +/// hr->requestGet(HttpRequest::DEFAULT_POLICY_ID, +/// 0, // Priority, not used yet +/// url, +/// NULL, // options +/// NULL, // additional headers +/// handler); +/// +/// If that returns a value other than LLCORE_HTTP_HANDLE_INVALID, +/// the request was successfully issued and there will eventally +/// be a status delivered to the handler. If invalid is returnedd, +/// the actual status can be retrieved by calling hr->getStatus(). +/// +/// Completing requests and delivering notifications. Operations +/// are all performed by the worker thread and will be driven to +/// completion regardless of caller actions. Notification of +/// completion (success or failure) is done by calls to +/// HttpRequest::update() which will invoke handlers for completed +/// requests: +/// +/// hr->update(0); +/// // Callbacks into handler->onCompleted() +/// +/// +/// Threads. +/// +/// Threads are supported and used by this library. The various +/// classes, methods and members are documented with thread +/// constraints which programmers must follow and which are +/// defined as follows: +/// +/// consumer Any thread that has instanced HttpRequest and is +/// issuing requests. A particular instance can only +/// be used by one consumer thread but a consumer may +/// have many instances available to it. +/// init Special consumer thread, usually the main thread, +/// involved in setting up the library at startup. +/// worker Thread used internally by the library to perform +/// HTTP operations. Consumers will not have to deal +/// with this thread directly but some APIs are reserved +/// to it. +/// any Consumer or worker thread. +/// +/// For the most part, API users will not have to do much in the +/// way of ensuring thread safely. However, there is a tremendous +/// amount of sharing between threads of read-only data. So when +/// documentation declares that an option or header instance +/// becomes shared between consumer and worker, the consumer must +/// not modify the shared object. +/// +/// Internally, there is almost no thread synchronization. During +/// normal operations (non-init, non-term), only the request queue +/// and the multiple reply queues are shared between threads and +/// only here are mutexes used. +/// #include "linden_common.h" // Modifies curl/curl.h interfaces @@ -239,9 +321,10 @@ struct HttpStatus return *this; } - static const type_enum_t EXT_CURL_EASY = 0; - static const type_enum_t EXT_CURL_MULTI = 1; - static const type_enum_t LLCORE = 2; + static const type_enum_t EXT_CURL_EASY = 0; ///< mStatus is an error from a curl_easy_*() call + static const type_enum_t EXT_CURL_MULTI = 1; ///< mStatus is an error from a curl_multi_*() call + static const type_enum_t LLCORE = 2; ///< mStatus is an HE_* error code + ///< 100-999 directly represent HTTP status codes type_enum_t mType; short mStatus; diff --git a/indra/llcorehttp/httprequest.cpp b/indra/llcorehttp/httprequest.cpp index 9b739a8825..7b1888e3eb 100755 --- a/indra/llcorehttp/httprequest.cpp +++ b/indra/llcorehttp/httprequest.cpp @@ -4,7 +4,7 @@ * * $LicenseInfo:firstyear=2012&license=viewerlgpl$ * Second Life Viewer Source Code - * Copyright (C) 2012, Linden Research, Inc. + * Copyright (C) 2012-2013, Linden Research, Inc. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -54,12 +54,8 @@ namespace LLCore // ==================================== -HttpRequest::policy_t HttpRequest::sNextPolicyID(1); - - HttpRequest::HttpRequest() - : //HttpHandler(), - mReplyQueue(NULL), + : mReplyQueue(NULL), mRequestQueue(NULL) { mRequestQueue = HttpRequestQueue::instanceOf(); @@ -90,45 +86,91 @@ HttpRequest::~HttpRequest() // ==================================== -HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, long value) +HttpRequest::policy_t HttpRequest::createPolicyClass() { if (HttpService::RUNNING == HttpService::instanceOf()->getState()) { - return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + return 0; } - return HttpService::instanceOf()->getGlobalOptions().set(opt, value); + return HttpService::instanceOf()->createPolicyClass(); } -HttpStatus HttpRequest::setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value) +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + long value, long * ret_value) { if (HttpService::RUNNING == HttpService::instanceOf()->getState()) { return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); } - return HttpService::instanceOf()->getGlobalOptions().set(opt, value); + return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value); } -HttpRequest::policy_t HttpRequest::createPolicyClass() +HttpStatus HttpRequest::setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, std::string * ret_value) { if (HttpService::RUNNING == HttpService::instanceOf()->getState()) { - return 0; + return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); } - return HttpService::instanceOf()->createPolicyClass(); + return HttpService::instanceOf()->setPolicyOption(opt, pclass, value, ret_value); } -HttpStatus HttpRequest::setPolicyClassOption(policy_t policy_id, - EClassPolicy opt, - long value) +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, + long value, HttpHandler * handler) { - if (HttpService::RUNNING == HttpService::instanceOf()->getState()) + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetGet * op = new HttpOpSetGet(); + if (! (status = op->setupSet(opt, pclass, value))) { - return HttpStatus(HttpStatus::LLCORE, HE_OPT_NOT_DYNAMIC); + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; + } + + mLastReqStatus = status; + handle = static_cast(op); + + return handle; +} + + +HttpHandle HttpRequest::setPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, HttpHandler * handler) +{ + HttpStatus status; + HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); + + HttpOpSetGet * op = new HttpOpSetGet(); + if (! (status = op->setupSet(opt, pclass, value))) + { + op->release(); + mLastReqStatus = status; + return handle; + } + op->setReplyPath(mReplyQueue, handler); + if (! (status = mRequestQueue->addOp(op))) // transfers refcount + { + op->release(); + mLastReqStatus = status; + return handle; } - return HttpService::instanceOf()->getClassOptions(policy_id).set(opt, value); + + mLastReqStatus = status; + handle = static_cast(op); + + return handle; } @@ -474,31 +516,6 @@ HttpHandle HttpRequest::requestSpin(int mode) return handle; } -// ==================================== -// Dynamic Policy Methods -// ==================================== - -HttpHandle HttpRequest::requestSetHttpProxy(const std::string & proxy, HttpHandler * handler) -{ - HttpStatus status; - HttpHandle handle(LLCORE_HTTP_HANDLE_INVALID); - - HttpOpSetGet * op = new HttpOpSetGet(); - op->setupSet(GP_HTTP_PROXY, proxy); - op->setReplyPath(mReplyQueue, handler); - if (! (status = mRequestQueue->addOp(op))) // transfers refcount - { - op->release(); - mLastReqStatus = status; - return handle; - } - - mLastReqStatus = status; - handle = static_cast(op); - - return handle; -} - } // end namespace LLCore diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 5000f47d0d..5c54d35a21 100755 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -56,6 +56,9 @@ class BufferArray; /// The class supports the current HTTP request operations: /// /// - requestGetByteRange: GET with Range header for a single range of bytes +/// - requestGet: +/// - requestPost: +/// - requestPut: /// /// Policy Classes /// @@ -100,9 +103,26 @@ public: /// Represents a default, catch-all policy class that guarantees /// eventual service for any HTTP request. - static const int DEFAULT_POLICY_ID = 0; + static const policy_t DEFAULT_POLICY_ID = 0; + static const policy_t INVALID_POLICY_ID = 0xFFFFFFFFU; + static const policy_t GLOBAL_POLICY_ID = 0xFFFFFFFEU; - enum EGlobalPolicy + /// Create a new policy class into which requests can be made. + /// + /// All class creation must occur before threads are started and + /// transport begins. Policy classes are limited to a small value. + /// Currently that limit is the default class + 1. + /// + /// @return If positive, the policy_id used to reference + /// the class in other methods. If 0, requests + /// for classes have exceeded internal limits + /// or caller has tried to create a class after + /// threads have been started. Caller must fallback + /// and recover. + /// + static policy_t createPolicyClass(); + + enum EPolicyOption { /// Maximum number of connections the library will use to /// perform operations. This is somewhat soft as the underlying @@ -113,24 +133,30 @@ public: /// a somewhat soft value. There may be an additional five /// connections per policy class depending upon runtime /// behavior. - GP_CONNECTION_LIMIT, + /// + /// Both global and per-class + PO_CONNECTION_LIMIT, + + /// Limits the number of connections used for a single + /// literal address/port pair within the class. + PO_PER_HOST_CONNECTION_LIMIT, /// String containing a system-appropriate directory name /// where SSL certs are stored. - GP_CA_PATH, + PO_CA_PATH, /// String giving a full path to a file containing SSL certs. - GP_CA_FILE, + PO_CA_FILE, /// String of host/port to use as simple HTTP proxy. This is /// going to change in the future into something more elaborate /// that may support richer schemes. - GP_HTTP_PROXY, + PO_HTTP_PROXY, /// Long value that if non-zero enables the use of the /// traditional LLProxy code for http/socks5 support. If - /// enabled, has priority over GP_HTTP_PROXY. - GP_LLPROXY, + // enabled, has priority over GP_HTTP_PROXY. + PO_LLPROXY, /// Long value setting the logging trace level for the /// library. Possible values are: @@ -143,57 +169,46 @@ public: /// These values are also used in the trace modes for /// individual requests in HttpOptions. Also be aware that /// tracing tends to impact performance of the viewer. - GP_TRACE - }; - - /// Set a parameter on a global policy option. Calls - /// made after the start of the servicing thread are - /// not honored and return an error status. - /// - /// @param opt Enum of option to be set. - /// @param value Desired value of option. - /// @return Standard status code. - static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, long value); - static HttpStatus setPolicyGlobalOption(EGlobalPolicy opt, const std::string & value); - - /// Create a new policy class into which requests can be made. - /// - /// All class creation must occur before threads are started and - /// transport begins. Policy classes are limited to a small value. - /// Currently that limit is the default class + 1. - /// - /// @return If positive, the policy_id used to reference - /// the class in other methods. If 0, requests - /// for classes have exceeded internal limits - /// or caller has tried to create a class after - /// threads have been started. Caller must fallback - /// and recover. - /// - static policy_t createPolicyClass(); - - enum EClassPolicy - { - /// Limits the number of connections used for the class. - CP_CONNECTION_LIMIT, - - /// Limits the number of connections used for a single - /// literal address/port pair within the class. - CP_PER_HOST_CONNECTION_LIMIT, + PO_TRACE, /// Suitable requests are allowed to pipeline on their /// connections when they ask for it. - CP_ENABLE_PIPELINING + PO_ENABLE_PIPELINING, + + PO_LAST // Always at end }; - + + /// Set a policy option for a global or class parameter at + /// startup time (prior to thread start). + /// + /// @param opt Enum of option to be set. + /// @param pclass For class-based options, the policy class ID to + /// be changed. For globals, specify GLOBAL_POLICY_ID. + /// @param value Desired value of option. + /// @param ret_value Pointer to receive effective set value + /// if successful. May be NULL if effective + /// value not wanted. + /// @return Standard status code. + static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + long value, long * ret_value); + static HttpStatus setStaticPolicyOption(EPolicyOption opt, policy_t pclass, + const std::string & value, std::string * ret_value); + /// Set a parameter on a class-based policy option. Calls /// made after the start of the servicing thread are /// not honored and return an error status. /// - /// @param policy_id ID of class as returned by @see createPolicyClass(). - /// @param opt Enum of option to be set. - /// @param value Desired value of option. - /// @return Standard status code. - static HttpStatus setPolicyClassOption(policy_t policy_id, EClassPolicy opt, long value); + /// @param opt Enum of option to be set. + /// @param pclass For class-based options, the policy class ID to + /// be changed. Ignored for globals but recommend + /// using INVALID_POLICY_ID in this case. + /// @param value Desired value of option. + /// @return Handle of dynamic request. Use @see getStatus() if + /// the returned handle is invalid. + HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, long value, + HttpHandler * handler); + HttpHandle setPolicyOption(EPolicyOption opt, policy_t pclass, const std::string & value, + HttpHandler * handler); /// @} @@ -495,16 +510,6 @@ public: /// @} - /// @name DynamicPolicyMethods - /// - /// @{ - - /// Request that a running transport pick up a new proxy setting. - /// An empty string will indicate no proxy is to be used. - HttpHandle requestSetHttpProxy(const std::string & proxy, HttpHandler * handler); - - /// @} - protected: void generateNotification(HttpOperation * op); @@ -526,7 +531,6 @@ private: /// Must be established before any threading is allowed to /// start. /// - static policy_t sNextPolicyID; /// @} // End Global State diff --git a/indra/llcorehttp/tests/test_httprequest.hpp b/indra/llcorehttp/tests/test_httprequest.hpp index 27d65f171e..f1b9c02393 100755 --- a/indra/llcorehttp/tests/test_httprequest.hpp +++ b/indra/llcorehttp/tests/test_httprequest.hpp @@ -1213,7 +1213,7 @@ void HttpRequestTestObjectType::test<12>() HttpRequest::createService(); // Enable tracing - HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL); // Start threading early so that thread memory is invariant // over the test. @@ -1331,7 +1331,7 @@ void HttpRequestTestObjectType::test<13>() HttpRequest::createService(); // Enable tracing - HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, 2); + HttpRequest::setStaticPolicyOption(HttpRequest::PO_TRACE, HttpRequest::DEFAULT_POLICY_ID, 2, NULL); // Start threading early so that thread memory is invariant // over the test. @@ -2972,6 +2972,142 @@ void HttpRequestTestObjectType::test<21>() } +template <> template <> +void HttpRequestTestObjectType::test<22>() +{ + ScopedCurlInit ready; + + set_test_name("HttpRequest GET 503s with 'Retry-After'"); + + // This tests mainly that the code doesn't fall over if + // various well- and mis-formed Retry-After headers are + // sent along with the response. Direct inspection of + // the parsing result isn't supported. + + // Handler can be stack-allocated *if* there are no dangling + // references to it after completion of this method. + // Create before memory record as the string copy will bump numbers. + TestHandler2 handler(this, "handler"); + std::string url_base(get_base_url() + "/503/"); // path to 503 generators + + // record the total amount of dynamically allocated memory + mMemTotal = GetMemTotal(); + mHandlerCalls = 0; + + HttpRequest * req = NULL; + HttpOptions * opts = NULL; + + try + { + // Get singletons created + HttpRequest::createService(); + + // Start threading early so that thread memory is invariant + // over the test. + HttpRequest::startThread(); + + // create a new ref counted object with an implicit reference + req = new HttpRequest(); + ensure("Memory allocated on construction", mMemTotal < GetMemTotal()); + + opts = new HttpOptions(); + opts->setRetries(1); // Retry once only + opts->setUseRetryAfter(true); // Try to parse the retry-after header + + // Issue a GET that 503s with valid retry-after + mStatus = HttpStatus(503); + int url_limit(6); + for (int i(0); i < url_limit; ++i) + { + std::ostringstream url; + url << url_base << i << "/"; + HttpHandle handle = req->requestGetByteRange(HttpRequest::DEFAULT_POLICY_ID, + 0U, + url.str(), + 0, + 0, + opts, + NULL, + &handler); + + std::ostringstream testtag; + testtag << "Valid handle returned for 503 request #" << i; + ensure(testtag.str(), handle != LLCORE_HTTP_HANDLE_INVALID); + } + + + // Run the notification pump. + int count(0); + int limit(300); // One retry but several seconds needed + while (count++ < limit && mHandlerCalls < url_limit) + { + req->update(0); + usleep(100000); + } + ensure("Request executed in reasonable time", count < limit); + ensure("One handler invocation for request", mHandlerCalls == url_limit); + + // Okay, request a shutdown of the servicing thread + mStatus = HttpStatus(); + mHandlerCalls = 0; + HttpHandle handle = req->requestStopThread(&handler); + ensure("Valid handle returned for second request", handle != LLCORE_HTTP_HANDLE_INVALID); + + // Run the notification pump again + count = 0; + limit = 100; + while (count++ < limit && mHandlerCalls < 1) + { + req->update(1000000); + usleep(100000); + } + ensure("Second request executed in reasonable time", count < limit); + ensure("Second handler invocation", mHandlerCalls == 1); + + // See that we actually shutdown the thread + count = 0; + limit = 10; + while (count++ < limit && ! HttpService::isStopped()) + { + usleep(100000); + } + ensure("Thread actually stopped running", HttpService::isStopped()); + + // release options + opts->release(); + opts = NULL; + + // release the request object + delete req; + req = NULL; + + // Shut down service + HttpRequest::destroyService(); + +#if defined(WIN32) + // Can only do this memory test on Windows. On other platforms, + // the LL logging system holds on to memory and produces what looks + // like memory leaks... + + // printf("Old mem: %d, New mem: %d\n", mMemTotal, GetMemTotal()); + ensure("Memory usage back to that at entry", mMemTotal == GetMemTotal()); +#endif + } + catch (...) + { + stop_thread(req); + if (opts) + { + opts->release(); + opts = NULL; + } + delete req; + HttpRequest::destroyService(); + throw; + } +} + + } // end namespace tut namespace diff --git a/indra/llcorehttp/tests/test_llcorehttp_peer.py b/indra/llcorehttp/tests/test_llcorehttp_peer.py index 75a3c39ef2..f6c4d1a820 100755 --- a/indra/llcorehttp/tests/test_llcorehttp_peer.py +++ b/indra/llcorehttp/tests/test_llcorehttp_peer.py @@ -9,7 +9,7 @@ $LicenseInfo:firstyear=2008&license=viewerlgpl$ Second Life Viewer Source Code -Copyright (C) 2012, Linden Research, Inc. +Copyright (C) 2012-2013, Linden Research, Inc. This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -47,6 +47,17 @@ from testrunner import freeport, run, debug, VERBOSE class TestHTTPRequestHandler(BaseHTTPRequestHandler): """This subclass of BaseHTTPRequestHandler is to receive and echo LLSD-flavored messages sent by the C++ LLHTTPClient. + + [Merge with viewer-cat later] + - '/503/' Generate 503 responses with various kinds + of 'retry-after' headers + -- '/503/0/' "Retry-After: 2" + -- '/503/1/' "Retry-After: Thu, 31 Dec 2043 23:59:59 GMT" + -- '/503/2/' "Retry-After: Fri, 31 Dec 1999 23:59:59 GMT" + -- '/503/3/' "Retry-After: " + -- '/503/4/' "Retry-After: (*#*(@*(@(")" + -- '/503/5/' "Retry-After: aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo" + -- '/503/6/' "Retry-After: 1 2 3 4 5 6 7 8 9 10" """ def read(self): # The following logic is adapted from the library module @@ -107,7 +118,41 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): if "/sleep/" in self.path: time.sleep(30) - if "fail" not in self.path: + if "/503/" in self.path: + # Tests for various kinds of 'Retry-After' header parsing + body = None + if "/503/0/" in self.path: + self.send_response(503) + self.send_header("retry-after", "2") + elif "/503/1/" in self.path: + self.send_response(503) + self.send_header("retry-after", "Thu, 31 Dec 2043 23:59:59 GMT") + elif "/503/2/" in self.path: + self.send_response(503) + self.send_header("retry-after", "Fri, 31 Dec 1999 23:59:59 GMT") + elif "/503/3/" in self.path: + self.send_response(503) + self.send_header("retry-after", "") + elif "/503/4/" in self.path: + self.send_response(503) + self.send_header("retry-after", "(*#*(@*(@(") + elif "/503/5/" in self.path: + self.send_response(503) + self.send_header("retry-after", "aklsjflajfaklsfaklfasfklasdfklasdgahsdhgasdiogaioshdgo") + elif "/503/6/" in self.path: + self.send_response(503) + self.send_header("retry-after", "1 2 3 4 5 6 7 8 9 10") + else: + # Unknown request + self.send_response(400) + body = "Unknown /503/ path in server" + if "/reflect/" in self.path: + self.reflect_headers() + self.send_header("Content-type", "text/plain") + self.end_headers() + if body: + self.wfile.write(body) + elif "fail" not in self.path: data = data.copy() # we're going to modify # Ensure there's a "reply" key in data, even if there wasn't before data["reply"] = data.get("reply", llsd.LLSD("success")) diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 0c242e57db..104debe023 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -105,8 +105,9 @@ void LLAppCoreHttp::init() } // Point to our certs or SSH/https: will fail on connect - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_CA_FILE, - gDirUtilp->getCAFile()); + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_CA_FILE, + LLCore::HttpRequest::GLOBAL_POLICY_ID, + gDirUtilp->getCAFile(), NULL); if (! status) { LL_ERRS("Init") << "Failed to set CA File for HTTP services. Reason: " << status.toString() @@ -114,7 +115,9 @@ void LLAppCoreHttp::init() } // Establish HTTP Proxy, if desired. - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_LLPROXY, 1); + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_LLPROXY, + LLCore::HttpRequest::GLOBAL_POLICY_ID, + 1, NULL); if (! status) { LL_WARNS("Init") << "Failed to set HTTP proxy for HTTP services. Reason: " << status.toString() @@ -131,7 +134,9 @@ void LLAppCoreHttp::init() { long trace_level(0L); trace_level = long(gSavedSettings.getU32(http_trace)); - status = LLCore::HttpRequest::setPolicyGlobalOption(LLCore::HttpRequest::GP_TRACE, trace_level); + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_TRACE, + LLCore::HttpRequest::GLOBAL_POLICY_ID, + trace_level, NULL); } // Setup default policy and constrain if directed to @@ -164,6 +169,9 @@ void LLAppCoreHttp::init() } } + // Need a request object to handle dynamic options before setting them + mRequest = new LLCore::HttpRequest; + // Apply initial settings refreshSettings(true); @@ -175,8 +183,6 @@ void LLAppCoreHttp::init() << LL_ENDL; } - mRequest = new LLCore::HttpRequest; - // Register signals for settings and state changes for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { @@ -287,12 +293,13 @@ void LLAppCoreHttp::refreshSettings(bool initial) // Set it and report // *TODO: These are intended to be per-host limits when we can // support that in llcorehttp/libcurl. - LLCore::HttpStatus status; - status = LLCore::HttpRequest::setPolicyClassOption(mPolicies[policy], - LLCore::HttpRequest::CP_CONNECTION_LIMIT, - setting); - if (! status) + LLCore::HttpHandle handle; + handle = mRequest->setPolicyOption(LLCore::HttpRequest::PO_CONNECTION_LIMIT, + mPolicies[policy], + setting, NULL); + if (LLCORE_HTTP_HANDLE_INVALID == handle) { + LLCore::HttpStatus status(mRequest->getStatus()); LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage << " concurrency. Reason: " << status.toString() << LL_ENDL; -- cgit v1.3 From f3927c6ca2aad757fe88fdd59b87986ca8b207a8 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Tue, 30 Jul 2013 15:21:31 -0400 Subject: SH-4371 Reduce 22mS inter-connection latency. This really extended into the client-side request throttling. Moved this from llmeshrepository (which doesn't really want to do connection management) into llcorehttp. It's now a class option with configurable rate. This still isn't the right thing to do as it creates coupling between viewer and services. When we get to pipelining, this notion becomes invalid. --- indra/llcorehttp/_httpinternal.h | 1 + indra/llcorehttp/_httppolicy.cpp | 67 ++++++++++++++++++++++++++++++++--- indra/llcorehttp/_httppolicyclass.cpp | 15 ++++++-- indra/llcorehttp/_httppolicyclass.h | 1 + indra/llcorehttp/_httpservice.cpp | 3 +- indra/llcorehttp/httprequest.h | 23 ++++++++++++ indra/newview/llappcorehttp.cpp | 31 ++++++++++++---- indra/newview/llmeshrepository.cpp | 59 ++++++++++-------------------- indra/newview/llmeshrepository.h | 11 +++--- 9 files changed, 152 insertions(+), 59 deletions(-) (limited to 'indra/newview/llappcorehttp.cpp') diff --git a/indra/llcorehttp/_httpinternal.h b/indra/llcorehttp/_httpinternal.h index 80f4f34942..effc6a42c5 100755 --- a/indra/llcorehttp/_httpinternal.h +++ b/indra/llcorehttp/_httpinternal.h @@ -143,6 +143,7 @@ const int HTTP_CONNECTION_LIMIT_MAX = 256; // Miscellaneous defaults const long HTTP_PIPELINING_DEFAULT = 0L; const bool HTTP_USE_RETRY_AFTER_DEFAULT = true; +const long HTTP_THROTTLE_RATE_DEFAULT = 0L; // Tuning parameters diff --git a/indra/llcorehttp/_httppolicy.cpp b/indra/llcorehttp/_httppolicy.cpp index 32a9ba282a..808eebc6cc 100755 --- a/indra/llcorehttp/_httppolicy.cpp +++ b/indra/llcorehttp/_httppolicy.cpp @@ -49,12 +49,18 @@ struct HttpPolicy::ClassState { public: ClassState() + : mThrottleEnd(0), + mThrottleLeft(0L), + mRequestCount(0L) {} HttpReadyQueue mReadyQueue; HttpRetryQueue mRetryQueue; HttpPolicyClass mOptions; + HttpTime mThrottleEnd; + long mThrottleLeft; + long mRequestCount; }; @@ -190,6 +196,13 @@ void HttpPolicy::retryOp(HttpOpRequest * op) // the worker thread may sleep hard otherwise will ask for // normal polling frequency. // +// Implements a client-side request rate throttle as well. +// This is intended to mimic and predict throttling behavior +// of grid services but that is difficult to do with different +// time bases. This also represents a rigid coupling between +// viewer and server that makes it hard to change parameters +// and I hope we can make this go away with pipelining. +// HttpService::ELoopSpeed HttpPolicy::processReadyQueue() { const HttpTime now(totalTime()); @@ -199,12 +212,22 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() for (int policy_class(0); policy_class < mClasses.size(); ++policy_class) { ClassState & state(*mClasses[policy_class]); + const bool throttle_enabled(state.mOptions.mThrottleRate > 0L); + const bool throttle_current(throttle_enabled && now < state.mThrottleEnd); + + if (throttle_current && state.mThrottleLeft <= 0) + { + // Throttled condition, don't serve this class but don't sleep hard. + result = HttpService::NORMAL; + continue; + } + int active(transport.getActiveCountInClass(policy_class)); int needed(state.mOptions.mConnectionLimit - active); // Expect negatives here HttpRetryQueue & retryq(state.mRetryQueue); HttpReadyQueue & readyq(state.mReadyQueue); - + if (needed > 0) { // First see if we have any retries... @@ -218,10 +241,27 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() op->stageFromReady(mService); op->release(); - + + ++state.mRequestCount; --needed; + if (throttle_enabled) + { + if (now >= state.mThrottleEnd) + { + // Throttle expired, move to next window + LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; + state.mThrottleLeft = state.mOptions.mThrottleRate; + state.mThrottleEnd = now + HttpTime(1000000); + } + if (--state.mThrottleLeft <= 0) + { + goto throttle_on; + } + } } - + // Now go on to the new requests... while (needed > 0 && ! readyq.empty()) { @@ -231,10 +271,29 @@ HttpService::ELoopSpeed HttpPolicy::processReadyQueue() op->stageFromReady(mService); op->release(); + ++state.mRequestCount; --needed; + if (throttle_enabled) + { + if (now >= state.mThrottleEnd) + { + // Throttle expired, move to next window + LL_DEBUGS("CoreHttp") << "Throttle expired with " << state.mThrottleLeft + << " requests to go and " << state.mRequestCount + << " requests issued." << LL_ENDL; + state.mThrottleLeft = state.mOptions.mThrottleRate; + state.mThrottleEnd = now + HttpTime(1000000); + } + if (--state.mThrottleLeft <= 0) + { + goto throttle_on; + } + } } } - + + throttle_on: + if (! readyq.empty() || ! retryq.empty()) { // If anything is ready, continue looping... diff --git a/indra/llcorehttp/_httppolicyclass.cpp b/indra/llcorehttp/_httppolicyclass.cpp index fe4359081a..f34a8e9f1e 100755 --- a/indra/llcorehttp/_httppolicyclass.cpp +++ b/indra/llcorehttp/_httppolicyclass.cpp @@ -36,7 +36,8 @@ namespace LLCore HttpPolicyClass::HttpPolicyClass() : mConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), mPerHostConnectionLimit(HTTP_CONNECTION_LIMIT_DEFAULT), - mPipelining(HTTP_PIPELINING_DEFAULT) + mPipelining(HTTP_PIPELINING_DEFAULT), + mThrottleRate(HTTP_THROTTLE_RATE_DEFAULT) {} @@ -51,6 +52,7 @@ HttpPolicyClass & HttpPolicyClass::operator=(const HttpPolicyClass & other) mConnectionLimit = other.mConnectionLimit; mPerHostConnectionLimit = other.mPerHostConnectionLimit; mPipelining = other.mPipelining; + mThrottleRate = other.mThrottleRate; } return *this; } @@ -59,7 +61,8 @@ HttpPolicyClass & HttpPolicyClass::operator=(const HttpPolicyClass & other) HttpPolicyClass::HttpPolicyClass(const HttpPolicyClass & other) : mConnectionLimit(other.mConnectionLimit), mPerHostConnectionLimit(other.mPerHostConnectionLimit), - mPipelining(other.mPipelining) + mPipelining(other.mPipelining), + mThrottleRate(other.mThrottleRate) {} @@ -79,6 +82,10 @@ HttpStatus HttpPolicyClass::set(HttpRequest::EPolicyOption opt, long value) mPipelining = llclamp(value, 0L, 1L); break; + case HttpRequest::PO_THROTTLE_RATE: + mThrottleRate = llclamp(value, 0L, 1000000L); + break; + default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } @@ -103,6 +110,10 @@ HttpStatus HttpPolicyClass::get(HttpRequest::EPolicyOption opt, long * value) co *value = mPipelining; break; + case HttpRequest::PO_THROTTLE_RATE: + *value = mThrottleRate; + break; + default: return HttpStatus(HttpStatus::LLCORE, HE_INVALID_ARG); } diff --git a/indra/llcorehttp/_httppolicyclass.h b/indra/llcorehttp/_httppolicyclass.h index 69fb459d22..38f1194ded 100755 --- a/indra/llcorehttp/_httppolicyclass.h +++ b/indra/llcorehttp/_httppolicyclass.h @@ -63,6 +63,7 @@ public: long mConnectionLimit; long mPerHostConnectionLimit; long mPipelining; + long mThrottleRate; }; // end class HttpPolicyClass } // end namespace LLCore diff --git a/indra/llcorehttp/_httpservice.cpp b/indra/llcorehttp/_httpservice.cpp index e21d196a3e..c94249dc2d 100755 --- a/indra/llcorehttp/_httpservice.cpp +++ b/indra/llcorehttp/_httpservice.cpp @@ -52,7 +52,8 @@ const HttpService::OptionDescriptor HttpService::sOptionDesc[] = { false, true, true, false }, // PO_HTTP_PROXY { true, true, true, false }, // PO_LLPROXY { true, true, true, false }, // PO_TRACE - { true, true, false, true } // PO_ENABLE_PIPELINING + { true, true, false, true }, // PO_ENABLE_PIPELINING + { true, true, false, true } // PO_THROTTLE_RATE }; HttpService * HttpService::sInstance(NULL); volatile HttpService::EState HttpService::sState(NOT_INITIALIZED); diff --git a/indra/llcorehttp/httprequest.h b/indra/llcorehttp/httprequest.h index 5c54d35a21..651654844a 100755 --- a/indra/llcorehttp/httprequest.h +++ b/indra/llcorehttp/httprequest.h @@ -139,23 +139,33 @@ public: /// Limits the number of connections used for a single /// literal address/port pair within the class. + /// + /// Per-class only PO_PER_HOST_CONNECTION_LIMIT, /// String containing a system-appropriate directory name /// where SSL certs are stored. + /// + /// Global only PO_CA_PATH, /// String giving a full path to a file containing SSL certs. + /// + /// Global only PO_CA_FILE, /// String of host/port to use as simple HTTP proxy. This is /// going to change in the future into something more elaborate /// that may support richer schemes. + /// + /// Global only PO_HTTP_PROXY, /// Long value that if non-zero enables the use of the /// traditional LLProxy code for http/socks5 support. If // enabled, has priority over GP_HTTP_PROXY. + /// + /// Global only PO_LLPROXY, /// Long value setting the logging trace level for the @@ -169,12 +179,25 @@ public: /// These values are also used in the trace modes for /// individual requests in HttpOptions. Also be aware that /// tracing tends to impact performance of the viewer. + /// + /// Global only PO_TRACE, /// Suitable requests are allowed to pipeline on their /// connections when they ask for it. + /// + /// Per-class only PO_ENABLE_PIPELINING, + /// Controls whether client-side throttling should be + /// performed on this policy class. Positive values + /// enable throttling and specify the request rate + /// (requests per second) that should be targetted. + /// A value of zero, the default, specifies no throttling. + /// + /// Per-class only + PO_THROTTLE_RATE, + PO_LAST // Always at end }; diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 104debe023..b90b9749f9 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -40,32 +40,33 @@ static const struct U32 mMin; U32 mMax; U32 mDivisor; + U32 mRate; std::string mKey; const char * mUsage; } init_data[] = // Default and dynamic values for classes { { - LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 1, + LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 1, 0, "TextureFetchConcurrency", "texture fetch" }, { - LLAppCoreHttp::AP_MESH1, 32, 1, 128, 1, + LLAppCoreHttp::AP_MESH1, 32, 1, 128, 1, 100, "MeshMaxConcurrentRequests", "mesh fetch" }, { - LLAppCoreHttp::AP_MESH2, 8, 1, 32, 4, + LLAppCoreHttp::AP_MESH2, 8, 1, 32, 4, 100, "MeshMaxConcurrentRequests", "mesh2 fetch" }, { - LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 1, + LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 1, 0, "", "large mesh fetch" }, { - LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 1, + LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 1, 0, "", "asset upload" } @@ -267,10 +268,28 @@ void LLAppCoreHttp::cleanup() void LLAppCoreHttp::refreshSettings(bool initial) { + LLCore::HttpStatus status; + for (int i(0); i < LL_ARRAY_SIZE(init_data); ++i) { const EAppPolicy policy(init_data[i].mPolicy); + // Set any desired throttle + if (initial && init_data[i].mRate) + { + // Init-time only, can use the static setters here + status = LLCore::HttpRequest::setStaticPolicyOption(LLCore::HttpRequest::PO_THROTTLE_RATE, + mPolicies[policy], + init_data[i].mRate, + NULL); + if (! status) + { + LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage + << " throttle rate. Reason: " << status.toString() + << LL_ENDL; + } + } + // Get target connection concurrency value U32 setting(init_data[i].mDefault); if (! init_data[i].mKey.empty() && gSavedSettings.controlExists(init_data[i].mKey)) @@ -299,7 +318,7 @@ void LLAppCoreHttp::refreshSettings(bool initial) setting, NULL); if (LLCORE_HTTP_HANDLE_INVALID == handle) { - LLCore::HttpStatus status(mRequest->getStatus()); + status = mRequest->getStatus(); LL_WARNS("Init") << "Unable to set " << init_data[i].mUsage << " concurrency. Reason: " << status.toString() << LL_ENDL; diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index d02384f87c..e9b1a10e73 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -229,7 +229,6 @@ LLMeshRepository gMeshRepo; const S32 MESH_HEADER_SIZE = 4096; // Important: assumption is that headers fit in this space -const U32 MAX_MESH_REQUESTS_PER_SECOND = 100; const S32 REQUEST_HIGH_WATER_MIN = 32; const S32 REQUEST_HIGH_WATER_MAX = 80; const S32 REQUEST_LOW_WATER_MIN = 16; @@ -613,7 +612,6 @@ void log_upload_error(LLCore::HttpStatus status, const LLSD& content, LLMeshRepoThread::LLMeshRepoThread() : LLThread("mesh repo"), mWaiting(false), - mHttpRetries(0U), mHttpRequest(NULL), mHttpOptions(NULL), mHttpLargeOptions(NULL), @@ -701,23 +699,9 @@ void LLMeshRepoThread::run() if (! LLApp::isQuitting()) { - static U32 count = 0; - static F32 last_hundred = gFrameTimeSeconds; - - if (gFrameTimeSeconds - last_hundred > 1.f) - { //a second has gone by, clear count - last_hundred = gFrameTimeSeconds; - count = 0; - } - else - { - count += mHttpRetries; - } - mHttpRetries = 0U; - - // NOTE: throttling intentionally favors LOD requests over header requests + // NOTE: order of queue processing intentionally favors LOD requests over header requests - while (!mLODReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && mHttpRequestSet.size() < sRequestHighWater) + while (!mLODReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) { if (! mMutex) { @@ -728,7 +712,7 @@ void LLMeshRepoThread::run() mLODReqQ.pop(); LLMeshRepository::sLODProcessing--; mMutex->unlock(); - if (!fetchMeshLOD(req.mMeshParams, req.mLOD, count))//failed, resubmit + if (!fetchMeshLOD(req.mMeshParams, req.mLOD))//failed, resubmit { mMutex->lock(); mLODReqQ.push(req) ; @@ -737,7 +721,7 @@ void LLMeshRepoThread::run() } } - while (!mHeaderReqQ.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && mHttpRequestSet.size() < sRequestHighWater) + while (!mHeaderReqQ.empty() && mHttpRequestSet.size() < sRequestHighWater) { if (! mMutex) { @@ -747,7 +731,7 @@ void LLMeshRepoThread::run() HeaderRequest req = mHeaderReqQ.front(); mHeaderReqQ.pop(); mMutex->unlock(); - if (!fetchMeshHeader(req.mMeshParams, count))//failed, resubmit + if (!fetchMeshHeader(req.mMeshParams))//failed, resubmit { mMutex->lock(); mHeaderReqQ.push(req) ; @@ -762,14 +746,14 @@ void LLMeshRepoThread::run() // order will lose. Keep to the throttle enforcement and pay // attention to the highwater level (enforced in each fetchXXX() // method). - if (! mSkinRequests.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && mHttpRequestSet.size() < sRequestHighWater) + if (! mSkinRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) { // *FIXME: this really does need a lock as do the following ones std::set incomplete; for (std::set::iterator iter = mSkinRequests.begin(); iter != mSkinRequests.end(); ++iter) { LLUUID mesh_id = *iter; - if (!fetchMeshSkinInfo(mesh_id, count)) + if (!fetchMeshSkinInfo(mesh_id)) { incomplete.insert(mesh_id); } @@ -777,13 +761,13 @@ void LLMeshRepoThread::run() mSkinRequests.swap(incomplete); } - if (! mDecompositionRequests.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && mHttpRequestSet.size() < sRequestHighWater) + if (! mDecompositionRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) { std::set incomplete; for (std::set::iterator iter = mDecompositionRequests.begin(); iter != mDecompositionRequests.end(); ++iter) { LLUUID mesh_id = *iter; - if (!fetchMeshDecomposition(mesh_id, count)) + if (!fetchMeshDecomposition(mesh_id)) { incomplete.insert(mesh_id); } @@ -791,13 +775,13 @@ void LLMeshRepoThread::run() mDecompositionRequests.swap(incomplete); } - if (! mPhysicsShapeRequests.empty() && count < MAX_MESH_REQUESTS_PER_SECOND && mHttpRequestSet.size() < sRequestHighWater) + if (! mPhysicsShapeRequests.empty() && mHttpRequestSet.size() < sRequestHighWater) { std::set incomplete; for (std::set::iterator iter = mPhysicsShapeRequests.begin(); iter != mPhysicsShapeRequests.end(); ++iter) { LLUUID mesh_id = *iter; - if (!fetchMeshPhysicsShape(mesh_id, count)) + if (!fetchMeshPhysicsShape(mesh_id)) { incomplete.insert(mesh_id); } @@ -965,7 +949,7 @@ LLCore::HttpHandle LLMeshRepoThread::getByteRange(const std::string & url, int c } -bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id, U32& count) +bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id) { if (!mHeaderMutex) @@ -1023,7 +1007,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id, U32& count) } //reading from VFS failed for whatever reason, fetch from sim - if (count >= MAX_MESH_REQUESTS_PER_SECOND || mHttpRequestSet.size() >= sRequestHighWater) + if (mHttpRequestSet.size() >= sRequestHighWater) { return false; } @@ -1060,7 +1044,7 @@ bool LLMeshRepoThread::fetchMeshSkinInfo(const LLUUID& mesh_id, U32& count) return ret; } -bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id, U32& count) +bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id) { if (!mHeaderMutex) { @@ -1118,7 +1102,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id, U32& count) } //reading from VFS failed for whatever reason, fetch from sim - if (count >= MAX_MESH_REQUESTS_PER_SECOND || mHttpRequestSet.size() >= sRequestHighWater) + if (mHttpRequestSet.size() >= sRequestHighWater) { return false; } @@ -1155,7 +1139,7 @@ bool LLMeshRepoThread::fetchMeshDecomposition(const LLUUID& mesh_id, U32& count) return ret; } -bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id, U32& count) +bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id) { if (!mHeaderMutex) { @@ -1212,7 +1196,7 @@ bool LLMeshRepoThread::fetchMeshPhysicsShape(const LLUUID& mesh_id, U32& count) } //reading from VFS failed for whatever reason, fetch from sim - if (count >= MAX_MESH_REQUESTS_PER_SECOND || mHttpRequestSet.size() >= sRequestHighWater) + if (mHttpRequestSet.size() >= sRequestHighWater) { return false; } @@ -1282,7 +1266,7 @@ void LLMeshRepoThread::decActiveHeaderRequests() } //return false if failed to get header -bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& count) +bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params) { { //look for mesh in asset in vfs @@ -1331,7 +1315,6 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c handler->mHttpHandle = handle; mHttpRequestSet.insert(handler); ++LLMeshRepository::sHTTPRequestCount; - ++count; } } @@ -1339,7 +1322,7 @@ bool LLMeshRepoThread::fetchMeshHeader(const LLVolumeParams& mesh_params, U32& c } //return false if failed to get mesh lod. -bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count) +bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod) { if (!mHeaderMutex) { @@ -1413,7 +1396,6 @@ bool LLMeshRepoThread::fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, handler->mHttpHandle = handle; mHttpRequestSet.insert(handler); ++LLMeshRepository::sHTTPRequestCount; - ++count; } } else @@ -2374,11 +2356,8 @@ void LLMeshHandlerBase::onCompleted(LLCore::HttpHandle handle, LLCore::HttpRespo { mProcessed = true; - // Accumulate retries, we'll use these to offset the HTTP - // count and maybe hold to a throttle better. unsigned int retries(0U); response->getRetries(NULL, &retries); - gMeshRepo.mThread->mHttpRetries += retries; LLMeshRepository::sHTTPRetryCount += retries; LLCore::HttpStatus status(response->getStatus()); diff --git a/indra/newview/llmeshrepository.h b/indra/newview/llmeshrepository.h index 70079eed23..400ceb4ad7 100755 --- a/indra/newview/llmeshrepository.h +++ b/indra/newview/llmeshrepository.h @@ -322,7 +322,6 @@ public: // llcorehttp library interface objects. LLCore::HttpStatus mHttpStatus; - unsigned int mHttpRetries; LLCore::HttpRequest * mHttpRequest; LLCore::HttpOptions * mHttpOptions; LLCore::HttpOptions * mHttpLargeOptions; @@ -345,8 +344,8 @@ public: void lockAndLoadMeshLOD(const LLVolumeParams& mesh_params, S32 lod); void loadMeshLOD(const LLVolumeParams& mesh_params, S32 lod); - bool fetchMeshHeader(const LLVolumeParams& mesh_params, U32& count); - bool fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod, U32& count); + bool fetchMeshHeader(const LLVolumeParams& mesh_params); + bool fetchMeshLOD(const LLVolumeParams& mesh_params, S32 lod); bool headerReceived(const LLVolumeParams& mesh_params, U8* data, S32 data_size); bool lodReceived(const LLVolumeParams& mesh_params, S32 lod, U8* data, S32 data_size); bool skinInfoReceived(const LLUUID& mesh_id, U8* data, S32 data_size); @@ -363,15 +362,15 @@ public: //send request for skin info, returns true if header info exists // (should hold onto mesh_id and try again later if header info does not exist) - bool fetchMeshSkinInfo(const LLUUID& mesh_id, U32& count); + bool fetchMeshSkinInfo(const LLUUID& mesh_id); //send request for decomposition, returns true if header info exists // (should hold onto mesh_id and try again later if header info does not exist) - bool fetchMeshDecomposition(const LLUUID& mesh_id, U32& count); + bool fetchMeshDecomposition(const LLUUID& mesh_id); //send request for PhysicsShape, returns true if header info exists // (should hold onto mesh_id and try again later if header info does not exist) - bool fetchMeshPhysicsShape(const LLUUID& mesh_id, U32& count); + bool fetchMeshPhysicsShape(const LLUUID& mesh_id); static void incActiveLODRequests(); static void decActiveLODRequests(); -- cgit v1.3 From 549e20cf7766dc7fba2f42883659a6bcac863d91 Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Mon, 5 Aug 2013 13:54:14 -0400 Subject: Change the setting for GetMesh2 meshes to Mesh2MaxConcurrentRequests. While linking GetMesh2 to the old setting was simpler from a user point-of-view, they really shouldn't be linked and the old one will go away. This one may be renamed to AssetMaxConcurrentRequests or something similar if we get to the mesh/texture unification step. --- indra/newview/app_settings/settings.xml | 14 ++++++++++++-- indra/newview/llappcorehttp.cpp | 23 ++++++++++++++--------- indra/newview/llmeshrepository.cpp | 4 +--- 3 files changed, 27 insertions(+), 14 deletions(-) (limited to 'indra/newview/llappcorehttp.cpp') diff --git a/indra/newview/app_settings/settings.xml b/indra/newview/app_settings/settings.xml index 344079b640..770a8529f1 100755 --- a/indra/newview/app_settings/settings.xml +++ b/indra/newview/app_settings/settings.xml @@ -9708,11 +9708,21 @@ Value 16 - + Mesh2MaxConcurrentRequests + + Comment + Number of connections to use for loading meshes. + Persist + 1 + Type + U32 + Value + 8 + MeshMaxConcurrentRequests Comment - Number of threads to use for loading meshes. + Number of connections to use for loading meshes (legacy system). Persist 1 Type diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index b90b9749f9..01317fe32f 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -32,6 +32,13 @@ #include "llviewercontrol.h" +// Here is where we begin to get our connection usage under control. +// This establishes llcorehttp policy classes that, among other +// things, limit the maximum number of connections to outside +// services. Each of the entries below maps to a policy class and +// has a limit, sometimes configurable, of how many connections can +// be open at a time. + const F64 LLAppCoreHttp::MAX_THREAD_WAIT_TIME(10.0); static const struct { @@ -39,34 +46,33 @@ static const struct U32 mDefault; U32 mMin; U32 mMax; - U32 mDivisor; U32 mRate; std::string mKey; const char * mUsage; } init_data[] = // Default and dynamic values for classes { { - LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 1, 0, + LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 0, "TextureFetchConcurrency", "texture fetch" }, { - LLAppCoreHttp::AP_MESH1, 32, 1, 128, 1, 100, + LLAppCoreHttp::AP_MESH1, 32, 1, 128, 100, "MeshMaxConcurrentRequests", "mesh fetch" }, { - LLAppCoreHttp::AP_MESH2, 8, 1, 32, 4, 100, - "MeshMaxConcurrentRequests", + LLAppCoreHttp::AP_MESH2, 8, 1, 32, 100, + "Mesh2MaxConcurrentRequests", "mesh2 fetch" }, { - LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 1, 0, + LLAppCoreHttp::AP_LARGE_MESH, 2, 1, 8, 0, "", "large mesh fetch" }, { - LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 1, 0, + LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 0, "", "asset upload" } @@ -298,8 +304,7 @@ void LLAppCoreHttp::refreshSettings(bool initial) if (new_setting) { // Treat zero settings as an ask for default - setting = new_setting / init_data[i].mDivisor; - setting = llclamp(setting, init_data[i].mMin, init_data[i].mMax); + setting = llclamp(new_setting, init_data[i].mMin, init_data[i].mMax); } } diff --git a/indra/newview/llmeshrepository.cpp b/indra/newview/llmeshrepository.cpp index e9b1a10e73..7146c7f4f5 100755 --- a/indra/newview/llmeshrepository.cpp +++ b/indra/newview/llmeshrepository.cpp @@ -2897,9 +2897,7 @@ void LLMeshRepository::notifyLoadedMeshes() else { // GetMesh2 operation with keepalives, etc. - // *TODO: Logic here is replicated from llappcorehttp.cpp, should really - // unify this and keep it in one place only. - LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("MeshMaxConcurrentRequests") / 4; + LLMeshRepoThread::sMaxConcurrentRequests = gSavedSettings.getU32("Mesh2MaxConcurrentRequests"); LLMeshRepoThread::sRequestHighWater = llclamp(5 * S32(LLMeshRepoThread::sMaxConcurrentRequests), REQUEST_HIGH_WATER_MIN, REQUEST_HIGH_WATER_MAX); -- cgit v1.3 From 195d319f65239238577ae15c22188da3839ae5cf Mon Sep 17 00:00:00 2001 From: Monty Brandenberg Date: Wed, 18 Sep 2013 18:44:41 -0400 Subject: SH-4492 Create a useful README for llcorehttp Last bit for this release. Describe stream adapters and how to select a policy class. Slight changes to setup code to make reality reflect documentation. --- indra/llcorehttp/README.Linden | 205 ++++++++++++++++++++++++++++++++++++++-- indra/newview/llappcorehttp.cpp | 38 ++++---- indra/newview/llappcorehttp.h | 97 +++++++++++++++++++ 3 files changed, 316 insertions(+), 24 deletions(-) (limited to 'indra/newview/llappcorehttp.cpp') diff --git a/indra/llcorehttp/README.Linden b/indra/llcorehttp/README.Linden index e5ff824388..8d18ed1a11 100644 --- a/indra/llcorehttp/README.Linden +++ b/indra/llcorehttp/README.Linden @@ -1,13 +1,13 @@ -1. HTTP fetching in 15 Minutes +1. HTTP Fetching in 15 Minutes Let's start with a trivial working example. You'll need a throwaway build of the viewer. And we'll use indra/newview/llappviewer.cpp as - our host. + the host module for these hacks. - Add some needed headers: + First, add some headers: #include "httpcommon.h" @@ -182,7 +182,7 @@ { // There's some data. A BufferArray is a linked list // of buckets. We'll create a linear buffer and copy - // it into it. + // the data into it. size_t data_len = data->size(); char * data_blob = new char [data_len + 1]; data->read(0, data_blob, data_len); @@ -419,7 +419,7 @@ HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148 BufferArray. The core data representation for request and response bodies. In HTTP responses, it's fetched with the getBody() method - and may be NULL or non-NULL but zero length. All successful data + and may be NULL or non-NULL with zero length. All successful data handling should check both conditions before attempting to fetch data from the object. Data access model uses simple read/write semantics: @@ -429,8 +429,8 @@ HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148 * read() * write() - There is a more sophisticated stream adapter that extends these - methods and will be covered below. So, one way to retrieve data + (There is a more sophisticated stream adapter that extends these + methods and will be covered below.) So, one way to retrieve data from a request is as follows: @@ -449,6 +449,7 @@ HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148 by just writing new values to the shared object. And in tests everything will appear to work. Then you ship and people in the real world start hitting read/write races in strings and then crash. + Don't be lazy. HttpHandle. Uniquely identifies a request and can be used to identify it in an onCompleted() method or cancel it if it's still @@ -459,9 +460,197 @@ HttpOperation::addAsReply: TRACE, ToReplyQueue, Handle: 086D3148 5. And Still More Refinements + (Note: The following refinements are just code fragments. They + don't directly fit into the working example above. But they + demonstrate several idioms you'll want to copy.) -6. Choosing a Policy Class + LLSD, std::streambuf, std::iostream. The read(), write() and + append() methods may be adequate for your purposes. But we use a + lot of LLSD. Its interfaces aren't particularly compatible with + BufferArray. And so two adapters are available to give + stream-like behaviors: BufferArrayStreamBuf and BufferArrayStream, + which implement the std::streambuf and std::iostream interfaces, + respectively. + + A std::streambuf interface isn't something you'll want to use + directly. Instead, you'll use the much friendlier std::iostream + interface found in BufferArrayStream. This adapter gives you all + the '>>' and '<<' operators you'll want as well as working + directly with the LLSD conversion operators. + + Some new headers: + + + #include "bufferstream.h" + #include "llsdserialize.h" + + + And an updated fragment based on onCompleted() above: + + + // Successful request. Try to fetch the data + LLCore::BufferArray * data = response->getBody(); + LLSD resp_llsd; + + if (data && data->size()) + { + // There's some data and we expect this to be + // LLSD. Checking of content type and validation + // during parsing would be admirable additions. + // But we'll forgo that now. + LLCore::BufferArrayStream data_stream(data); + LLSDSerialize::fromXML(resp_llsd, data_stream); + } + LL_INFOS("Hack") << "LLSD Received: " << resp_llsd << LL_ENDL; + } + else + { + + + Converting an LLSD object into an XML stream stored in a + BufferArray is just the reverse of the above: + BufferArray * data = new BufferArray(); + LLCore::BufferArrayStream data_stream(data); + + LLSD src_llsd; + src_llsd["foo"] = "bar"; + + LLSDSerialize::toXML(src_llsd, data_stream); + + // 'data' now contains an XML payload and can be sent + // to a web service using the requestPut() or requestPost() + // methods. + ... requestPost(...); + + // And don't forget to release the BufferArray. + data->release(); + data = NULL; + + + LLSD will often go hand-in-hand with BufferArray and data + transport. But you can also do all the streaming I/O you'd expect + of a std::iostream object: + + + BufferArray * data = new BufferArray(); + LLCore::BufferArrayStream data_stream(data); + + data_stream << "Hello, World!" << 29.4 << '\n'; + std::string str; + data_stream >> str; + std::cout << str << std::endl; + + data->release(); + // Actual delete will occur when 'data_stream' + // falls out of scope and is destructed. + + + Scoping objects and cleaning up. The examples haven't bothered + with cleanup of objects that are no longer needed. Instead, most + objects have been allocated as if they were global and eternal. + You'll put the objects in more appropriate feature objects and + clean them up as a group. Here's a checklist for actions you may + need to take on cleanup: + + * Call delete on: + o HttpHandlers created on the heap + o HttpRequest objects + * Call release() on: + o BufferArray objects + o HttpHeaders objects + o HttpOptions objects + o HttpResponse objects + + On program exit, as threads wind down, the library continues to + operate safely. Threads don't interact via the library and even + dangling references to HttpHandler objects are safe. If you don't + call HttpRequest::update(), handler references are never + dereferenced. + + You can take a more thorough approach to wind-down. Keep a list + of HttpHandles (not HttpHandlers) of outstanding requests. For + each of these, call HttpRequest::requestCancel() to cancel the + operation. (Don't add the cancel requests' handled to the list.) + This will cancel the outstanding requests that haven't completed. + Canceled or completed, all requests will queue notifications. You + can now cycle calling update() discarding responses. Continue + until all requests notify or a few seconds have passed. + + Global startup and shutdown is handled in the viewer. But you can + learn about it in the code or in the documentation in the headers. + + +6. Choosing a Policy Class + + Now it's time to get rid of the default policy class. Take a look + at the policy class definitions in newview/llappcorehttp.h. + Ideally, you'll find one that's compatible with what you're doing. + Some of the compatibility guidelines are: + + * Destination: Pair of host and port. Mixing requests with + different destinations may cause more connection setup and tear + down. + + * Method: http or https. Usually moot given destination. But + mixing these may also cause connection churn. + + * Transfer size: If you're moving 100MB at a time and you make your + requests to the same policy class as a lot of small, fast event + information that fast traffic is going to get stuck behind you + and someone's experience is going to be miserable. + + * Long poll requests: These are long-lived, must- do operations. + They have a special home called AP_LONG_POLL. + + * Concurrency: High concurrency (5 or more) and large transfer + sizes are incompatible. Another head-of-the-line problem. High + concurrency is tolerated when it's desired to get maximal + throughput. Mesh and texture downloads, for example. + + * Pipelined: If your requests are not idempotent, stay away from + anything marked 'soon' or 'yes'. Hidden retries may be a + problem for you. For now, would also recommend keeping PUT and + POST requests out of classes that may be pipelined. Support for + that is still a bit new. + + If you haven't found a compatible match, you can either create a + new class (llappcorehttp.*) or just use AP_DEFAULT, the catchall + class when all else fails. Inventory query operations might be a + candidate for a new class that supported pipelining on https:. + Same with display name lookups and other bursty-at-login + operations. For other things, AP_DEFAULT will do what it can and + will, in some way or another, tolerate any usage. Whether the + users' experiences are good are for you to determine. + + 7. FAQ + Q1. What do these policy classes achieve? + + A1. Previously, HTTP-using code in the viewer was written as if + it were some isolated, local operation that didn't have to + consider resources, contention or impact on services and the + larger environment. The result was an application with on the + order of 100 HTTP launch points in its codebase that could create + dozens or even 100's of TCP connections zeroing in on grid + services and disrupting networking equipment, web services and + innocent users. The use of policy classes (modeled on + http://en.wikipedia.org/wiki/Class-based_queueing) is a means to + restrict connection concurrency, good and necessary in itself. In + turn, that reduces demands on an expensive resource (connection + setup and concurrency) which relieves strain on network points. + That enables connection keepalive and opportunites for true + improvements in throughput and user experience. + + Another aspect of the classes is that they give some control over + how competing demands for the network will be apportioned. If + mesh fetches, texture fetches and inventory queries are all being + made at once, the relative weights of their classes' concurrency + limits established that apportioning. We now have an opportunity + to balance the entire viewer system. + + Q2. How's that data sharing with refcounts working for you? + + A2. Meh. diff --git a/indra/newview/llappcorehttp.cpp b/indra/newview/llappcorehttp.cpp index 01317fe32f..70dcffefb2 100755 --- a/indra/newview/llappcorehttp.cpp +++ b/indra/newview/llappcorehttp.cpp @@ -51,6 +51,11 @@ static const struct const char * mUsage; } init_data[] = // Default and dynamic values for classes { + { + LLAppCoreHttp::AP_DEFAULT, 8, 8, 8, 0, + "", + "other" + }, { LLAppCoreHttp::AP_TEXTURE, 8, 1, 12, 0, "TextureFetchConcurrency", @@ -75,6 +80,11 @@ static const struct LLAppCoreHttp::AP_UPLOADS, 2, 1, 8, 0, "", "asset upload" + }, + { + LLAppCoreHttp::AP_LONG_POLL, 32, 32, 32, 0, + "", + "long poll" } }; @@ -154,25 +164,21 @@ void LLAppCoreHttp::init() { const EAppPolicy policy(init_data[i].mPolicy); - // Create a policy class but use default for texture for now. - // This also has the side-effect of initializing the default - // class to desired values. - if (AP_TEXTURE == policy) + if (AP_DEFAULT == policy) { - mPolicies[policy] = mPolicies[AP_DEFAULT]; + // Pre-created + continue; } - else + + mPolicies[policy] = LLCore::HttpRequest::createPolicyClass(); + if (! mPolicies[policy]) { - mPolicies[policy] = LLCore::HttpRequest::createPolicyClass(); - if (! mPolicies[policy]) - { - // Use default policy (but don't accidentally modify default) - LL_WARNS("Init") << "Failed to create HTTP policy class for " << init_data[i].mUsage - << ". Using default policy." - << LL_ENDL; - mPolicies[policy] = mPolicies[AP_DEFAULT]; - continue; - } + // Use default policy (but don't accidentally modify default) + LL_WARNS("Init") << "Failed to create HTTP policy class for " << init_data[i].mUsage + << ". Using default policy." + << LL_ENDL; + mPolicies[policy] = mPolicies[AP_DEFAULT]; + continue; } } diff --git a/indra/newview/llappcorehttp.h b/indra/newview/llappcorehttp.h index 6dc3bb2130..40e3042b84 100755 --- a/indra/newview/llappcorehttp.h +++ b/indra/newview/llappcorehttp.h @@ -45,12 +45,109 @@ public: enum EAppPolicy { + /// Catchall policy class. Not used yet + /// but will have a generous concurrency + /// limit. Deep queueing possible by having + /// a chatty HTTP user. + /// + /// Destination: anywhere + /// Protocol: http: or https: + /// Transfer size: KB-MB + /// Long poll: no + /// Concurrency: high + /// Request rate: unknown + /// Pipelined: no AP_DEFAULT, + + /// Texture fetching policy class. Used to + /// download textures via capability or SSA + /// baking service. Deep queueing of requests. + /// Do not share. + /// + /// Destination: simhost:12046 & bake-texture:80 + /// Protocol: http: + /// Transfer size: KB-MB + /// Long poll: no + /// Concurrency: high + /// Request rate: high + /// Pipelined: soon AP_TEXTURE, + + /// Legacy mesh fetching policy class. Used to + /// download textures via 'GetMesh' capability. + /// To be deprecated. Do not share. + /// + /// Destination: simhost:12046 + /// Protocol: http: + /// Transfer size: KB-MB + /// Long poll: no + /// Concurrency: dangerously high + /// Request rate: high + /// Pipelined: no AP_MESH1, + + /// New mesh fetching policy class. Used to + /// download textures via 'GetMesh2' capability. + /// Used when fetch request (typically one LOD) + /// is 'small', currently defined as 2MB. + /// Very deeply queued. Do not share. + /// + /// Destination: simhost:12046 + /// Protocol: http: + /// Transfer size: KB-MB + /// Long poll: no + /// Concurrency: high + /// Request rate: high + /// Pipelined: soon AP_MESH2, + + /// Large mesh fetching policy class. Used to + /// download textures via 'GetMesh' or 'GetMesh2' + /// capability. Used when fetch request + /// is not small to avoid head-of-line problem + /// when large requests block a sequence of small, + /// fast requests. Can be shared with similar + /// traffic that can wait for longish stalls + /// (default timeout 600S). + /// + /// Destination: simhost:12046 + /// Protocol: http: + /// Transfer size: MB + /// Long poll: no + /// Concurrency: low + /// Request rate: low + /// Pipelined: soon AP_LARGE_MESH, + + /// Asset upload policy class. Used to store + /// assets (mesh only at the moment) via + /// changeable URL. Responses may take some + /// time (default timeout 240S). + /// + /// Destination: simhost:12043 + /// Protocol: https: + /// Transfer size: KB-MB + /// Long poll: no + /// Concurrency: low + /// Request rate: low + /// Pipelined: no AP_UPLOADS, + + /// Long-poll-type HTTP requests. Not + /// bound by a connection limit. Requests + /// will typically hang around for a long + /// time (~30S). Only shareable with other + /// long-poll requests. + /// + /// Destination: simhost:12043 + /// Protocol: https: + /// Transfer size: KB + /// Long poll: yes + /// Concurrency: unlimited but low in practice + /// Request rate: low + /// Pipelined: no + AP_LONG_POLL, + AP_COUNT // Must be last }; -- cgit v1.3