From 3efe5b493442f2b90ffbb571d8fa24c8ab17bf5e Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Tue, 4 Mar 2025 19:55:55 +0200 Subject: #3644 Fix new logging of packet drops hitting performance --- indra/newview/llviewerthrottle.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'indra/newview/llviewerthrottle.cpp') diff --git a/indra/newview/llviewerthrottle.cpp b/indra/newview/llviewerthrottle.cpp index b0a00c29a4..dce85bcb03 100644 --- a/indra/newview/llviewerthrottle.cpp +++ b/indra/newview/llviewerthrottle.cpp @@ -304,6 +304,7 @@ void LLViewerThrottle::updateDynamicThrottle() } mUpdateTimer.reset(); + // Todo: account for dropped packets from LLPacketRing (or make the thing threaded) LLUnit mean_packets_lost = LLViewerStats::instance().getRecording().getMean(LLStatViewer::PACKETS_LOST_PERCENT); if (mean_packets_lost > TIGHTEN_THROTTLE_THRESHOLD) { -- cgit v1.3 From 5c2a331c44424722f069cd7a6b167f24de1afb83 Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Wed, 5 Mar 2025 10:53:17 +0200 Subject: #3644 Fix new logging of packet drops hitting performance #2 Misunderstood how mDroppedPackets worked, clear out what's not needed --- indra/llmessage/llpacketring.cpp | 2 ++ indra/newview/llappviewer.cpp | 1 - indra/newview/llviewerstats.cpp | 2 -- indra/newview/llviewerstats.h | 1 - indra/newview/llviewerthrottle.cpp | 1 - indra/newview/llworld.cpp | 2 -- 6 files changed, 2 insertions(+), 7 deletions(-) (limited to 'indra/newview/llviewerthrottle.cpp') diff --git a/indra/llmessage/llpacketring.cpp b/indra/llmessage/llpacketring.cpp index 470398152c..da3c502e9d 100644 --- a/indra/llmessage/llpacketring.cpp +++ b/indra/llmessage/llpacketring.cpp @@ -304,6 +304,8 @@ S32 LLPacketRing::drainSocket(S32 socket) S32 num_dropped_packets = (num_loops - 1 + old_num_packets) - mNumBufferedPackets; if (num_dropped_packets > 0) { + // It will eventually be accounted by mDroppedPackets + // and mPacketsLost, but track it here for logging purposes. mNumDroppedPackets += num_dropped_packets; } return (S32)(mNumBufferedPackets); diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index c351f63e85..e8546d422b 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -3413,7 +3413,6 @@ LLSD LLAppViewer::getViewerInfo() const info["PACKETS_LOST"] = LLViewerStats::instance().getRecording().getSum(LLStatViewer::PACKETS_LOST); info["PACKETS_IN"] = packets_in; info["PACKETS_PCT"] = 100.f*info["PACKETS_LOST"].asReal() / info["PACKETS_IN"].asReal(); - info["PACKETS_DROPPED"] = LLViewerStats::instance().getRecording().getSum(LLStatViewer::PACKETS_DROPPED); } if (mServerReleaseNotesURL.empty()) diff --git a/indra/newview/llviewerstats.cpp b/indra/newview/llviewerstats.cpp index 4f52ab644c..73aabf49d1 100644 --- a/indra/newview/llviewerstats.cpp +++ b/indra/newview/llviewerstats.cpp @@ -121,7 +121,6 @@ LLTrace::CountStatHandle<> FPS("FPS", "Frames rendered"), PACKETS_IN("Packets In", "Packets received"), PACKETS_LOST("packetsloststat", "Packets lost"), PACKETS_OUT("packetsoutstat", "Packets sent"), - PACKETS_DROPPED("packetsdropped", "Packets dropped"), TEXTURE_PACKETS("texturepacketsstat", "Texture data packets received"), CHAT_COUNT("chatcount", "Chat messages sent"), IM_COUNT("imcount", "IMs sent"), @@ -648,7 +647,6 @@ void send_viewer_stats(bool include_preferences) fail["send_packet"] = (S32) gMessageSystem->mSendPacketFailureCount; fail["dropped"] = (S32) gMessageSystem->mDroppedPackets; - fail["ring_dropped"] = (S32)gMessageSystem->mPacketRing.getNumDroppedPackets(); fail["resent"] = (S32) gMessageSystem->mResentPackets; fail["failed_resends"] = (S32) gMessageSystem->mFailedResendPackets; fail["off_circuit"] = (S32) gMessageSystem->mOffCircuitPackets; diff --git a/indra/newview/llviewerstats.h b/indra/newview/llviewerstats.h index 4cab2b48a5..8aed1c537e 100644 --- a/indra/newview/llviewerstats.h +++ b/indra/newview/llviewerstats.h @@ -119,7 +119,6 @@ extern LLTrace::CountStatHandle<> FPS, PACKETS_IN, PACKETS_LOST, PACKETS_OUT, - PACKETS_DROPPED, TEXTURE_PACKETS, CHAT_COUNT, IM_COUNT, diff --git a/indra/newview/llviewerthrottle.cpp b/indra/newview/llviewerthrottle.cpp index dce85bcb03..b0a00c29a4 100644 --- a/indra/newview/llviewerthrottle.cpp +++ b/indra/newview/llviewerthrottle.cpp @@ -304,7 +304,6 @@ void LLViewerThrottle::updateDynamicThrottle() } mUpdateTimer.reset(); - // Todo: account for dropped packets from LLPacketRing (or make the thing threaded) LLUnit mean_packets_lost = LLViewerStats::instance().getRecording().getMean(LLStatViewer::PACKETS_LOST_PERCENT); if (mean_packets_lost > TIGHTEN_THROTTLE_THRESHOLD) { diff --git a/indra/newview/llworld.cpp b/indra/newview/llworld.cpp index a1eed0340b..899733ccc3 100644 --- a/indra/newview/llworld.cpp +++ b/indra/newview/llworld.cpp @@ -795,7 +795,6 @@ void LLWorld::updateNetStats() S32 packets_in = gMessageSystem->mPacketsIn - mLastPacketsIn; S32 packets_out = gMessageSystem->mPacketsOut - mLastPacketsOut; S32 packets_lost = gMessageSystem->mDroppedPackets - mLastPacketsLost; - S32 ring_packets_dropped = gMessageSystem->mPacketRing.getNumDroppedPackets(); F64Bits actual_in_bits(gMessageSystem->mPacketRing.getAndResetActualInBits()); F64Bits actual_out_bits(gMessageSystem->mPacketRing.getAndResetActualOutBits()); @@ -806,7 +805,6 @@ void LLWorld::updateNetStats() add(LLStatViewer::PACKETS_IN, packets_in); add(LLStatViewer::PACKETS_OUT, packets_out); add(LLStatViewer::PACKETS_LOST, packets_lost); - add(LLStatViewer::PACKETS_DROPPED, ring_packets_dropped); F32 total_packets_in = (F32)LLViewerStats::instance().getRecording().getSum(LLStatViewer::PACKETS_IN); if (total_packets_in > 0.f) -- cgit v1.3 From c99e3167ed8549bc13d7df03b1e12dc15b0a080f Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Thu, 13 Mar 2025 23:55:38 +0200 Subject: #3644 Adjust throttle based of how busy buffer is --- indra/llmessage/llpacketring.cpp | 6 ++++++ indra/llmessage/llpacketring.h | 1 + indra/llmessage/message.h | 4 +--- indra/newview/llappviewer.cpp | 1 + indra/newview/llviewerthrottle.cpp | 14 +++++++++++--- indra/newview/llviewerthrottle.h | 3 +++ 6 files changed, 23 insertions(+), 6 deletions(-) (limited to 'indra/newview/llviewerthrottle.cpp') diff --git a/indra/llmessage/llpacketring.cpp b/indra/llmessage/llpacketring.cpp index da3c502e9d..eb6650c6c5 100644 --- a/indra/llmessage/llpacketring.cpp +++ b/indra/llmessage/llpacketring.cpp @@ -344,6 +344,12 @@ bool LLPacketRing::expandRing() return true; } +F32 LLPacketRing::getBufferLoadRate() const +{ + // goes up to MAX_BUFFER_RING_SIZE + return (F32)mNumBufferedPackets / (F32)DEFAULT_BUFFER_RING_SIZE; +} + void LLPacketRing::dumpPacketRingStats() { mNumDroppedPacketsTotal += mNumDroppedPackets; diff --git a/indra/llmessage/llpacketring.h b/indra/llmessage/llpacketring.h index 237efc12e0..572dcbd271 100644 --- a/indra/llmessage/llpacketring.h +++ b/indra/llmessage/llpacketring.h @@ -64,6 +64,7 @@ public: S32 getNumBufferedBytes() const { return mNumBufferedBytes; } S32 getNumDroppedPackets() const { return mNumDroppedPacketsTotal + mNumDroppedPackets; } + F32 getBufferLoadRate() const; // from 0 to 4 (0 - empty, 1 - default size is full) void dumpPacketRingStats(); protected: // returns 'true' if we should intentionally drop a packet diff --git a/indra/llmessage/message.h b/indra/llmessage/message.h index 1844d5e7cd..30945cac51 100644 --- a/indra/llmessage/message.h +++ b/indra/llmessage/message.h @@ -538,7 +538,6 @@ public: //void buildMessage(); - S32 zeroCode(U8 **data, S32 *data_size); S32 zeroCodeExpand(U8 **data, S32 *data_size); S32 zeroCodeAdjustCurrentSendTotal(); @@ -755,6 +754,7 @@ public: S32 getReceiveBytes() const; S32 getUnackedListSize() const { return mUnackedListSize; } + F32 getBufferLoadRate() const { return mPacketRing.getBufferLoadRate(); } //const char* getCurrentSMessageName() const { return mCurrentSMessageName; } //const char* getCurrentSBlockName() const { return mCurrentSBlockName; } @@ -842,12 +842,10 @@ private: LLUUID mSessionID; void addTemplate(LLMessageTemplate *templatep); - bool decodeTemplate( const U8* buffer, S32 buffer_size, LLMessageTemplate** msg_template ); void logMsgFromInvalidCircuit( const LLHost& sender, bool recv_reliable ); void logTrustedMsgFromUntrustedCircuit( const LLHost& sender ); void logValidMsg(LLCircuitData *cdp, const LLHost& sender, bool recv_reliable, bool recv_resent, bool recv_acks ); - void logRanOffEndOfPacket( const LLHost& sender ); class LLMessageCountInfo { diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index a05c2376a0..4cf651de33 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5443,6 +5443,7 @@ void LLAppViewer::idleNetwork() // Retransmit unacknowledged packets. gXferManager->retransmitUnackedPackets(); gAssetStorage->checkForTimeouts(); + gViewerThrottle.setBufferLoadRate(gMessageSystem->getBufferLoadRate()); gViewerThrottle.updateDynamicThrottle(); // Check that the circuit between the viewer and the agent's current diff --git a/indra/newview/llviewerthrottle.cpp b/indra/newview/llviewerthrottle.cpp index b0a00c29a4..8d935e4243 100644 --- a/indra/newview/llviewerthrottle.cpp +++ b/indra/newview/llviewerthrottle.cpp @@ -48,6 +48,8 @@ const F32 MIN_FRACTIONAL = 0.2f; const F32 MIN_BANDWIDTH = 50.f; const F32 MAX_BANDWIDTH = 6000.f; const F32 STEP_FRACTIONAL = 0.1f; +const F32 HIGH_BUFFER_LOAD_TRESHOLD = 1.f; +const F32 LOW_BUFFER_LOAD_TRESHOLD = 0.8f; const LLUnit TIGHTEN_THROTTLE_THRESHOLD(3.0f); // packet loss % per s const LLUnit EASE_THROTTLE_THRESHOLD(0.5f); // packet loss % per s const F32 DYNAMIC_UPDATE_DURATION = 5.0f; // seconds @@ -146,7 +148,7 @@ LLViewerThrottleGroup LLViewerThrottleGroup::operator-(const LLViewerThrottleGro void LLViewerThrottleGroup::sendToSim() const { - LL_INFOS() << "Sending throttle settings, total BW " << mThrottleTotal << LL_ENDL; + LL_DEBUGS("Throttle") << "Sending throttle settings, total BW " << mThrottleTotal << LL_ENDL; LLMessageSystem* msg = gMessageSystem; msg->newMessageFast(_PREHASH_AgentThrottle); @@ -305,7 +307,10 @@ void LLViewerThrottle::updateDynamicThrottle() mUpdateTimer.reset(); LLUnit mean_packets_lost = LLViewerStats::instance().getRecording().getMean(LLStatViewer::PACKETS_LOST_PERCENT); - if (mean_packets_lost > TIGHTEN_THROTTLE_THRESHOLD) + if ( + mean_packets_lost > TIGHTEN_THROTTLE_THRESHOLD // already losing packets + || mBufferLoadRate >= HIGH_BUFFER_LOAD_TRESHOLD // let viewer sort through the backlog before it starts dropping packets + ) { if (mThrottleFrac <= MIN_FRACTIONAL || mCurrentBandwidth / 1024.0f <= MIN_BANDWIDTH) { @@ -318,7 +323,8 @@ void LLViewerThrottle::updateDynamicThrottle() mCurrent.sendToSim(); LL_INFOS() << "Tightening network throttle to " << mCurrentBandwidth << LL_ENDL; } - else if (mean_packets_lost <= EASE_THROTTLE_THRESHOLD) + else if (mean_packets_lost <= EASE_THROTTLE_THRESHOLD + && mBufferLoadRate < LOW_BUFFER_LOAD_TRESHOLD) { if (mThrottleFrac >= MAX_FRACTIONAL || mCurrentBandwidth / 1024.0f >= MAX_BANDWIDTH) { @@ -331,4 +337,6 @@ void LLViewerThrottle::updateDynamicThrottle() mCurrent.sendToSim(); LL_INFOS() << "Easing network throttle to " << mCurrentBandwidth << LL_ENDL; } + + mBufferLoadRate = 0; } diff --git a/indra/newview/llviewerthrottle.h b/indra/newview/llviewerthrottle.h index 28a24d04fc..9973c88549 100644 --- a/indra/newview/llviewerthrottle.h +++ b/indra/newview/llviewerthrottle.h @@ -70,12 +70,15 @@ public: void updateDynamicThrottle(); void resetDynamicThrottle(); + void setBufferLoadRate(F32 rate) { mBufferLoadRate = llmax(mBufferLoadRate, rate); } + LLViewerThrottleGroup getThrottleGroup(const F32 bandwidth_kbps); static const std::string sNames[TC_EOF]; protected: F32 mMaxBandwidth; F32 mCurrentBandwidth; + F32 mBufferLoadRate = 0; LLViewerThrottleGroup mCurrent; -- cgit v1.3