From 6d0b0a77eed584c6bcf77f6cd5fdbdbf89a987d7 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Tue, 18 Feb 2025 11:38:52 -0800 Subject: drain UDP socket to avoid dropped packets (#3565) drain UDP socket in idleNetwork() to avoid dropped packets --- indra/llmessage/llpacketring.cpp | 444 ++++++++++++++++++--------------------- 1 file changed, 208 insertions(+), 236 deletions(-) (limited to 'indra/llmessage/llpacketring.cpp') diff --git a/indra/llmessage/llpacketring.cpp b/indra/llmessage/llpacketring.cpp index be838770a8..ae5a2168db 100644 --- a/indra/llmessage/llpacketring.cpp +++ b/indra/llmessage/llpacketring.cpp @@ -1,6 +1,6 @@ /** * @file llpacketring.cpp - * @brief implementation of LLPacketRing class for a packet. + * @brief implementation of LLPacketRing class. * * $LicenseInfo:firstyear=2001&license=viewerlgpl$ * Second Life Viewer Source Code @@ -43,329 +43,301 @@ #include "message.h" #include "u64.h" -/////////////////////////////////////////////////////////// -LLPacketRing::LLPacketRing () : - mUseInThrottle(false), - mUseOutThrottle(false), - mInThrottle(256000.f), - mOutThrottle(64000.f), - mActualBitsIn(0), - mActualBitsOut(0), - mMaxBufferLength(64000), - mInBufferLength(0), - mOutBufferLength(0), - mDropPercentage(0.0f), - mPacketsToDrop(0x0) +constexpr S16 MAX_BUFFER_RING_SIZE = 1024; +constexpr S16 DEFAULT_BUFFER_RING_SIZE = 256; + +LLPacketRing::LLPacketRing () + : mPacketRing(DEFAULT_BUFFER_RING_SIZE, nullptr) { + LLHost invalid_host; + for (size_t i = 0; i < mPacketRing.size(); ++i) + { + mPacketRing[i] = new LLPacketBuffer(invalid_host, nullptr, 0); + } } -/////////////////////////////////////////////////////////// LLPacketRing::~LLPacketRing () { - cleanup(); + for (auto packet : mPacketRing) + { + delete packet; + } + mPacketRing.clear(); + mNumBufferedPackets = 0; + mNumBufferedBytes = 0; + mHeadIndex = 0; } -/////////////////////////////////////////////////////////// -void LLPacketRing::cleanup () +S32 LLPacketRing::receivePacket (S32 socket, char *datap) { - LLPacketBuffer *packetp; + bool drop = computeDrop(); + return (mNumBufferedPackets > 0) ? + receiveOrDropBufferedPacket(datap, drop) : + receiveOrDropPacket(socket, datap, drop); +} - while (!mReceiveQueue.empty()) +bool send_packet_helper(int socket, const char * datap, S32 data_size, LLHost host) +{ + if (!LLProxy::isSOCKSProxyEnabled()) { - packetp = mReceiveQueue.front(); - delete packetp; - mReceiveQueue.pop(); + return send_packet(socket, datap, data_size, host.getAddress(), host.getPort()); } - while (!mSendQueue.empty()) - { - packetp = mSendQueue.front(); - delete packetp; - mSendQueue.pop(); - } -} + char headered_send_buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; -/////////////////////////////////////////////////////////// -void LLPacketRing::dropPackets (U32 num_to_drop) -{ - mPacketsToDrop += num_to_drop; -} + proxywrap_t *socks_header = static_cast(static_cast(&headered_send_buffer)); + socks_header->rsv = 0; + socks_header->addr = host.getAddress(); + socks_header->port = htons(host.getPort()); + socks_header->atype = ADDRESS_IPV4; + socks_header->frag = 0; -/////////////////////////////////////////////////////////// -void LLPacketRing::setDropPercentage (F32 percent_to_drop) -{ - mDropPercentage = percent_to_drop; -} + memcpy(headered_send_buffer + SOCKS_HEADER_SIZE, datap, data_size); -void LLPacketRing::setUseInThrottle(const bool use_throttle) -{ - mUseInThrottle = use_throttle; + return send_packet( socket, + headered_send_buffer, + data_size + SOCKS_HEADER_SIZE, + LLProxy::getInstance()->getUDPProxy().getAddress(), + LLProxy::getInstance()->getUDPProxy().getPort()); } -void LLPacketRing::setUseOutThrottle(const bool use_throttle) +bool LLPacketRing::sendPacket(int socket, const char * datap, S32 data_size, LLHost host) { - mUseOutThrottle = use_throttle; + mActualBytesOut += data_size; + return send_packet_helper(socket, datap, data_size, host); } -void LLPacketRing::setInBandwidth(const F32 bps) +void LLPacketRing::dropPackets (U32 num_to_drop) { - mInThrottle.setRate(bps); + mPacketsToDrop += num_to_drop; } -void LLPacketRing::setOutBandwidth(const F32 bps) +void LLPacketRing::setDropPercentage (F32 percent_to_drop) { - mOutThrottle.setRate(bps); + mDropPercentage = percent_to_drop; } -/////////////////////////////////////////////////////////// -S32 LLPacketRing::receiveFromRing (S32 socket, char *datap) -{ - if (mInThrottle.checkOverflow(0)) - { - // We don't have enough bandwidth, don't give them a packet. - return 0; - } - - LLPacketBuffer *packetp = NULL; - if (mReceiveQueue.empty()) +bool LLPacketRing::computeDrop() +{ + bool drop= (mDropPercentage > 0.0f && (ll_frand(100.f) < mDropPercentage)); + if (drop) { - // No packets on the queue, don't give them any. - return 0; + ++mPacketsToDrop; } - - S32 packet_size = 0; - packetp = mReceiveQueue.front(); - mReceiveQueue.pop(); - packet_size = packetp->getSize(); - if (packetp->getData() != NULL) + if (mPacketsToDrop > 0) { - memcpy(datap, packetp->getData(), packet_size); /*Flawfinder: ignore*/ + --mPacketsToDrop; + drop = true; } - // need to set sender IP/port!! - mLastSender = packetp->getHost(); - mLastReceivingIF = packetp->getReceivingInterface(); - delete packetp; - - this->mInBufferLength -= packet_size; - - // Adjust the throttle - mInThrottle.throttleOverflow(packet_size * 8.f); - return packet_size; + return drop; } -/////////////////////////////////////////////////////////// -S32 LLPacketRing::receivePacket (S32 socket, char *datap) +S32 LLPacketRing::receiveOrDropPacket(S32 socket, char *datap, bool drop) { S32 packet_size = 0; - // If using the throttle, simulate a limited size input buffer. - if (mUseInThrottle) + // pull straight from socket + if (LLProxy::isSOCKSProxyEnabled()) { - bool done = false; - - // push any current net packet (if any) onto delay ring - while (!done) + char buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; /* Flawfinder ignore */ + packet_size = receive_packet(socket, buffer); + if (packet_size > 0) { - LLPacketBuffer *packetp; - packetp = new LLPacketBuffer(socket); - - if (packetp->getSize()) - { - mActualBitsIn += packetp->getSize() * 8; - - // Fake packet loss - if (mDropPercentage && (ll_frand(100.f) < mDropPercentage)) - { - mPacketsToDrop++; - } - - if (mPacketsToDrop) - { - delete packetp; - packetp = NULL; - packet_size = 0; - mPacketsToDrop--; - } - } + mActualBytesIn += packet_size; + } - // If we faked packet loss, then we don't have a packet - // to use for buffer overflow testing - if (packetp) + if (packet_size > SOCKS_HEADER_SIZE) + { + if (drop) { - if (mInBufferLength + packetp->getSize() > mMaxBufferLength) - { - // Toss it. - LL_WARNS() << "Throwing away packet, overflowing buffer" << LL_ENDL; - delete packetp; - packetp = NULL; - } - else if (packetp->getSize()) - { - mReceiveQueue.push(packetp); - mInBufferLength += packetp->getSize(); - } - else - { - delete packetp; - packetp = NULL; - done = true; - } + packet_size = 0; } else - { - // No packetp, keep going? - no packetp == faked packet loss - } - } - - // Now, grab data off of the receive queue according to our - // throttled bandwidth settings. - packet_size = receiveFromRing(socket, datap); - } - else - { - // no delay, pull straight from net - if (LLProxy::isSOCKSProxyEnabled()) - { - U8 buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; - packet_size = receive_packet(socket, static_cast(static_cast(buffer))); - - if (packet_size > SOCKS_HEADER_SIZE) { // *FIX We are assuming ATYP is 0x01 (IPv4), not 0x03 (hostname) or 0x04 (IPv6) - memcpy(datap, buffer + SOCKS_HEADER_SIZE, packet_size - SOCKS_HEADER_SIZE); + packet_size -= SOCKS_HEADER_SIZE; // The unwrapped packet size + memcpy(datap, buffer + SOCKS_HEADER_SIZE, packet_size); proxywrap_t * header = static_cast(static_cast(buffer)); mLastSender.setAddress(header->addr); mLastSender.setPort(ntohs(header->port)); - - packet_size -= SOCKS_HEADER_SIZE; // The unwrapped packet size - } - else - { - packet_size = 0; + mLastReceivingIF = ::get_receiving_interface(); } } else { - packet_size = receive_packet(socket, datap); - mLastSender = ::get_sender(); + packet_size = 0; } - - mLastReceivingIF = ::get_receiving_interface(); - - if (packet_size) // did we actually get a packet? + } + else + { + packet_size = receive_packet(socket, datap); + if (packet_size > 0) { - if (mDropPercentage && (ll_frand(100.f) < mDropPercentage)) + mActualBytesIn += packet_size; + if (drop) { - mPacketsToDrop++; + packet_size = 0; } - - if (mPacketsToDrop) + else { - packet_size = 0; - mPacketsToDrop--; + mLastSender = ::get_sender(); + mLastReceivingIF = ::get_receiving_interface(); } } } - return packet_size; } -bool LLPacketRing::sendPacket(int h_socket, char * send_buffer, S32 buf_size, LLHost host) +S32 LLPacketRing::receiveOrDropBufferedPacket(char *datap, bool drop) { - bool status = true; - if (!mUseOutThrottle) + assert(mNumBufferedPackets > 0); + S32 packet_size = 0; + + S16 ring_size = (S16)(mPacketRing.size()); + S16 packet_index = (mHeadIndex + ring_size - mNumBufferedPackets) % ring_size; + LLPacketBuffer* packet = mPacketRing[packet_index]; + packet_size = packet->getSize(); + mLastSender = packet->getHost(); + mLastReceivingIF = packet->getReceivingInterface(); + + --mNumBufferedPackets; + mNumBufferedBytes -= packet_size; + if (mNumBufferedPackets == 0) { - return sendPacketImpl(h_socket, send_buffer, buf_size, host ); + assert(mNumBufferedBytes == 0); + } + + if (!drop) + { + assert(packet_size > 0); + memcpy(datap, packet->getData(), packet_size); } else { - mActualBitsOut += buf_size * 8; - LLPacketBuffer *packetp = NULL; - // See if we've got enough throttle to send a packet. - while (!mOutThrottle.checkOverflow(0.f)) - { - // While we have enough bandwidth, send a packet from the queue or the current packet + packet_size = 0; + } + return packet_size; +} + +S32 LLPacketRing::bufferInboundPacket(S32 socket) +{ + if (mNumBufferedPackets == mPacketRing.size() && mNumBufferedPackets < MAX_BUFFER_RING_SIZE) + { + expandRing(); + } - S32 packet_size = 0; - if (!mSendQueue.empty()) + LLPacketBuffer* packet = mPacketRing[mHeadIndex]; + S32 old_packet_size = packet->getSize(); + S32 packet_size = 0; + if (LLProxy::isSOCKSProxyEnabled()) + { + char buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; /* Flawfinder ignore */ + packet_size = receive_packet(socket, buffer); + if (packet_size > 0) + { + mActualBytesIn += packet_size; + if (packet_size > SOCKS_HEADER_SIZE) { - // Send a packet off of the queue - LLPacketBuffer *packetp = mSendQueue.front(); - mSendQueue.pop(); + // *FIX We are assuming ATYP is 0x01 (IPv4), not 0x03 (hostname) or 0x04 (IPv6) - mOutBufferLength -= packetp->getSize(); - packet_size = packetp->getSize(); + proxywrap_t * header = static_cast(static_cast(buffer)); + LLHost sender; + sender.setAddress(header->addr); + sender.setPort(ntohs(header->port)); - status = sendPacketImpl(h_socket, packetp->getData(), packet_size, packetp->getHost()); + packet_size -= SOCKS_HEADER_SIZE; // The unwrapped packet size + packet->init(buffer + SOCKS_HEADER_SIZE, packet_size, sender); - delete packetp; - // Update the throttle - mOutThrottle.throttleOverflow(packet_size * 8.f); + mHeadIndex = (mHeadIndex + 1) % (S16)(mPacketRing.size()); + if (mNumBufferedPackets < MAX_BUFFER_RING_SIZE) + { + ++mNumBufferedPackets; + mNumBufferedBytes += packet_size; + } + else + { + // we overwrote an older packet + mNumBufferedBytes += packet_size - old_packet_size; + } } else { - // If the queue's empty, we can just send this packet right away. - status = sendPacketImpl(h_socket, send_buffer, buf_size, host ); - packet_size = buf_size; - - // Update the throttle - mOutThrottle.throttleOverflow(packet_size * 8.f); - - // This was the packet we're sending now, there are no other packets - // that we need to send - return status; + packet_size = 0; } - - } - - // We haven't sent the incoming packet, add it to the queue - if (mOutBufferLength + buf_size > mMaxBufferLength) - { - // Nuke this packet, we overflowed the buffer. - // Toss it. - LL_WARNS() << "Throwing away outbound packet, overflowing buffer" << LL_ENDL; } - else + } + else + { + packet->init(socket); + packet_size = packet->getSize(); + if (packet_size > 0) { - static LLTimer queue_timer; - if ((mOutBufferLength > 4192) && queue_timer.getElapsedTimeF32() > 1.f) + mActualBytesIn += packet_size; + + mHeadIndex = (mHeadIndex + 1) % (S16)(mPacketRing.size()); + if (mNumBufferedPackets < MAX_BUFFER_RING_SIZE) { - // Add it to the queue - LL_INFOS() << "Outbound packet queue " << mOutBufferLength << " bytes" << LL_ENDL; - queue_timer.reset(); + ++mNumBufferedPackets; + mNumBufferedBytes += packet_size; + } + else + { + // we overwrote an older packet + mNumBufferedBytes += packet_size - old_packet_size; } - packetp = new LLPacketBuffer(host, send_buffer, buf_size); - - mOutBufferLength += packetp->getSize(); - mSendQueue.push(packetp); } } - - return status; + return packet_size; } -bool LLPacketRing::sendPacketImpl(int h_socket, const char * send_buffer, S32 buf_size, LLHost host) +S32 LLPacketRing::drainSocket(S32 socket) { - - if (!LLProxy::isSOCKSProxyEnabled()) + // drain into buffer + S32 packet_size = 1; + S32 num_loops = 0; + S32 old_num_packets = mNumBufferedPackets; + while (packet_size > 0) { - return send_packet(h_socket, send_buffer, buf_size, host.getAddress(), host.getPort()); + packet_size = bufferInboundPacket(socket); + ++num_loops; } + S32 num_dropped_packets = (num_loops - 1 + old_num_packets) - mNumBufferedPackets; + if (num_dropped_packets > 0) + { + LL_WARNS("Messaging") << "dropped " << num_dropped_packets << " UDP packets" << LL_ENDL; + } + return (S32)(mNumBufferedPackets); +} - char headered_send_buffer[NET_BUFFER_SIZE + SOCKS_HEADER_SIZE]; +bool LLPacketRing::expandRing() +{ + // compute larger size + constexpr S16 BUFFER_RING_EXPANSION = 256; + S16 old_size = (S16)(mPacketRing.size()); + S16 new_size = llmin(old_size + BUFFER_RING_EXPANSION, MAX_BUFFER_RING_SIZE); + if (new_size == old_size) + { + // mPacketRing is already maxed out + return false; + } - proxywrap_t *socks_header = static_cast(static_cast(&headered_send_buffer)); - socks_header->rsv = 0; - socks_header->addr = host.getAddress(); - socks_header->port = htons(host.getPort()); - socks_header->atype = ADDRESS_IPV4; - socks_header->frag = 0; + // make a larger ring and copy packet pointers + std::vector new_ring(new_size, nullptr); + for (S16 i = 0; i < old_size; ++i) + { + S16 j = (mHeadIndex + i) % old_size; + new_ring[i] = mPacketRing[j]; + } - memcpy(headered_send_buffer + SOCKS_HEADER_SIZE, send_buffer, buf_size); + // allocate new packets for the remainder of new_ring + LLHost invalid_host; + for (S16 i = old_size; i < new_size; ++i) + { + new_ring[i] = new LLPacketBuffer(invalid_host, nullptr, 0); + } - return send_packet( h_socket, - headered_send_buffer, - buf_size + SOCKS_HEADER_SIZE, - LLProxy::getInstance()->getUDPProxy().getAddress(), - LLProxy::getInstance()->getUDPProxy().getPort()); + // swap the rings and reset mHeadIndex + mPacketRing.swap(new_ring); + mHeadIndex = mNumBufferedPackets; + return true; } -- cgit v1.3 From 3efe5b493442f2b90ffbb571d8fa24c8ab17bf5e Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Tue, 4 Mar 2025 19:55:55 +0200 Subject: #3644 Fix new logging of packet drops hitting performance --- indra/llmessage/llpacketring.cpp | 16 +++++++++++++++- indra/llmessage/llpacketring.h | 5 +++++ indra/llmessage/message.cpp | 1 + indra/newview/llappviewer.cpp | 1 + indra/newview/llfloaterlagmeter.cpp | 1 + indra/newview/llviewerstats.cpp | 2 ++ indra/newview/llviewerstats.h | 1 + indra/newview/llviewerthrottle.cpp | 1 + indra/newview/llworld.cpp | 3 +++ 9 files changed, 30 insertions(+), 1 deletion(-) (limited to 'indra/llmessage/llpacketring.cpp') diff --git a/indra/llmessage/llpacketring.cpp b/indra/llmessage/llpacketring.cpp index ae5a2168db..470398152c 100644 --- a/indra/llmessage/llpacketring.cpp +++ b/indra/llmessage/llpacketring.cpp @@ -304,7 +304,7 @@ S32 LLPacketRing::drainSocket(S32 socket) S32 num_dropped_packets = (num_loops - 1 + old_num_packets) - mNumBufferedPackets; if (num_dropped_packets > 0) { - LL_WARNS("Messaging") << "dropped " << num_dropped_packets << " UDP packets" << LL_ENDL; + mNumDroppedPackets += num_dropped_packets; } return (S32)(mNumBufferedPackets); } @@ -341,3 +341,17 @@ bool LLPacketRing::expandRing() mHeadIndex = mNumBufferedPackets; return true; } + +void LLPacketRing::dumpPacketRingStats() +{ + mNumDroppedPacketsTotal += mNumDroppedPackets; + LL_INFOS("Messaging") << "Packet ring stats: " << std::endl + << "Buffered packets: " << mNumBufferedPackets << std::endl + << "Buffered bytes: " << mNumBufferedBytes << std::endl + << "Dropped packets current: " << mNumDroppedPackets << std::endl + << "Dropped packets total: " << mNumDroppedPacketsTotal << std::endl + << "Dropped packets percentage: " << mDropPercentage << "%" << std::endl + << "Actual in bytes: " << mActualBytesIn << std::endl + << "Actual out bytes: " << mActualBytesOut << LL_ENDL; + mNumDroppedPackets = 0; +} diff --git a/indra/llmessage/llpacketring.h b/indra/llmessage/llpacketring.h index 0dff2c63b1..237efc12e0 100644 --- a/indra/llmessage/llpacketring.h +++ b/indra/llmessage/llpacketring.h @@ -62,6 +62,9 @@ public: S32 getNumBufferedPackets() const { return (S32)(mNumBufferedPackets); } S32 getNumBufferedBytes() const { return mNumBufferedBytes; } + S32 getNumDroppedPackets() const { return mNumDroppedPacketsTotal + mNumDroppedPackets; } + + void dumpPacketRingStats(); protected: // returns 'true' if we should intentionally drop a packet bool computeDrop(); @@ -80,6 +83,8 @@ protected: std::vector mPacketRing; S16 mHeadIndex { 0 }; S16 mNumBufferedPackets { 0 }; + S32 mNumDroppedPackets { 0 }; + S32 mNumDroppedPacketsTotal { 0 }; S32 mNumBufferedBytes { 0 }; S32 mActualBytesIn { 0 }; diff --git a/indra/llmessage/message.cpp b/indra/llmessage/message.cpp index c130b7a6db..ad1ff86807 100644 --- a/indra/llmessage/message.cpp +++ b/indra/llmessage/message.cpp @@ -724,6 +724,7 @@ bool LLMessageSystem::checkMessages(LockMessageChecker&, S64 frame_count ) // Check to see if we need to print debug info if ((mt_sec - mCircuitPrintTime) > mCircuitPrintFreq) { + mPacketRing.dumpPacketRingStats(); dumpCircuitInfo(); mCircuitPrintTime = mt_sec; } diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index e8546d422b..c351f63e85 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -3413,6 +3413,7 @@ LLSD LLAppViewer::getViewerInfo() const info["PACKETS_LOST"] = LLViewerStats::instance().getRecording().getSum(LLStatViewer::PACKETS_LOST); info["PACKETS_IN"] = packets_in; info["PACKETS_PCT"] = 100.f*info["PACKETS_LOST"].asReal() / info["PACKETS_IN"].asReal(); + info["PACKETS_DROPPED"] = LLViewerStats::instance().getRecording().getSum(LLStatViewer::PACKETS_DROPPED); } if (mServerReleaseNotesURL.empty()) diff --git a/indra/newview/llfloaterlagmeter.cpp b/indra/newview/llfloaterlagmeter.cpp index 28fa8dea9a..1377526f69 100644 --- a/indra/newview/llfloaterlagmeter.cpp +++ b/indra/newview/llfloaterlagmeter.cpp @@ -200,6 +200,7 @@ void LLFloaterLagMeter::determineNetwork() // the network handlers are de-synched from the rendering. F32Milliseconds client_frame_time = frame_recording.getPeriodMean(LLStatViewer::FRAME_STACKTIME); + // Todo: account for LLPacketRing dropped packets? viewer drops those when it can't keep up if(packet_loss >= mNetworkPacketLossCritical) { mNetworkButton->setImageUnselected(LLUI::getUIImage(LAG_CRITICAL_IMAGE_NAME)); diff --git a/indra/newview/llviewerstats.cpp b/indra/newview/llviewerstats.cpp index 73aabf49d1..4f52ab644c 100644 --- a/indra/newview/llviewerstats.cpp +++ b/indra/newview/llviewerstats.cpp @@ -121,6 +121,7 @@ LLTrace::CountStatHandle<> FPS("FPS", "Frames rendered"), PACKETS_IN("Packets In", "Packets received"), PACKETS_LOST("packetsloststat", "Packets lost"), PACKETS_OUT("packetsoutstat", "Packets sent"), + PACKETS_DROPPED("packetsdropped", "Packets dropped"), TEXTURE_PACKETS("texturepacketsstat", "Texture data packets received"), CHAT_COUNT("chatcount", "Chat messages sent"), IM_COUNT("imcount", "IMs sent"), @@ -647,6 +648,7 @@ void send_viewer_stats(bool include_preferences) fail["send_packet"] = (S32) gMessageSystem->mSendPacketFailureCount; fail["dropped"] = (S32) gMessageSystem->mDroppedPackets; + fail["ring_dropped"] = (S32)gMessageSystem->mPacketRing.getNumDroppedPackets(); fail["resent"] = (S32) gMessageSystem->mResentPackets; fail["failed_resends"] = (S32) gMessageSystem->mFailedResendPackets; fail["off_circuit"] = (S32) gMessageSystem->mOffCircuitPackets; diff --git a/indra/newview/llviewerstats.h b/indra/newview/llviewerstats.h index 8aed1c537e..4cab2b48a5 100644 --- a/indra/newview/llviewerstats.h +++ b/indra/newview/llviewerstats.h @@ -119,6 +119,7 @@ extern LLTrace::CountStatHandle<> FPS, PACKETS_IN, PACKETS_LOST, PACKETS_OUT, + PACKETS_DROPPED, TEXTURE_PACKETS, CHAT_COUNT, IM_COUNT, diff --git a/indra/newview/llviewerthrottle.cpp b/indra/newview/llviewerthrottle.cpp index b0a00c29a4..dce85bcb03 100644 --- a/indra/newview/llviewerthrottle.cpp +++ b/indra/newview/llviewerthrottle.cpp @@ -304,6 +304,7 @@ void LLViewerThrottle::updateDynamicThrottle() } mUpdateTimer.reset(); + // Todo: account for dropped packets from LLPacketRing (or make the thing threaded) LLUnit mean_packets_lost = LLViewerStats::instance().getRecording().getMean(LLStatViewer::PACKETS_LOST_PERCENT); if (mean_packets_lost > TIGHTEN_THROTTLE_THRESHOLD) { diff --git a/indra/newview/llworld.cpp b/indra/newview/llworld.cpp index e0d8c25731..a1eed0340b 100644 --- a/indra/newview/llworld.cpp +++ b/indra/newview/llworld.cpp @@ -795,6 +795,7 @@ void LLWorld::updateNetStats() S32 packets_in = gMessageSystem->mPacketsIn - mLastPacketsIn; S32 packets_out = gMessageSystem->mPacketsOut - mLastPacketsOut; S32 packets_lost = gMessageSystem->mDroppedPackets - mLastPacketsLost; + S32 ring_packets_dropped = gMessageSystem->mPacketRing.getNumDroppedPackets(); F64Bits actual_in_bits(gMessageSystem->mPacketRing.getAndResetActualInBits()); F64Bits actual_out_bits(gMessageSystem->mPacketRing.getAndResetActualOutBits()); @@ -805,6 +806,7 @@ void LLWorld::updateNetStats() add(LLStatViewer::PACKETS_IN, packets_in); add(LLStatViewer::PACKETS_OUT, packets_out); add(LLStatViewer::PACKETS_LOST, packets_lost); + add(LLStatViewer::PACKETS_DROPPED, ring_packets_dropped); F32 total_packets_in = (F32)LLViewerStats::instance().getRecording().getSum(LLStatViewer::PACKETS_IN); if (total_packets_in > 0.f) @@ -838,6 +840,7 @@ void LLWorld::printPacketsLost() << " packets lost: " << cdp->getPacketsLost() << LL_ENDL; } } + LL_INFOS() << "Packets dropped by Packet Ring: " << gMessageSystem->mPacketRing.getNumDroppedPackets() << LL_ENDL; } void LLWorld::processCoarseUpdate(LLMessageSystem* msg, void** user_data) -- cgit v1.3 From 5c2a331c44424722f069cd7a6b167f24de1afb83 Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Wed, 5 Mar 2025 10:53:17 +0200 Subject: #3644 Fix new logging of packet drops hitting performance #2 Misunderstood how mDroppedPackets worked, clear out what's not needed --- indra/llmessage/llpacketring.cpp | 2 ++ indra/newview/llappviewer.cpp | 1 - indra/newview/llviewerstats.cpp | 2 -- indra/newview/llviewerstats.h | 1 - indra/newview/llviewerthrottle.cpp | 1 - indra/newview/llworld.cpp | 2 -- 6 files changed, 2 insertions(+), 7 deletions(-) (limited to 'indra/llmessage/llpacketring.cpp') diff --git a/indra/llmessage/llpacketring.cpp b/indra/llmessage/llpacketring.cpp index 470398152c..da3c502e9d 100644 --- a/indra/llmessage/llpacketring.cpp +++ b/indra/llmessage/llpacketring.cpp @@ -304,6 +304,8 @@ S32 LLPacketRing::drainSocket(S32 socket) S32 num_dropped_packets = (num_loops - 1 + old_num_packets) - mNumBufferedPackets; if (num_dropped_packets > 0) { + // It will eventually be accounted by mDroppedPackets + // and mPacketsLost, but track it here for logging purposes. mNumDroppedPackets += num_dropped_packets; } return (S32)(mNumBufferedPackets); diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index c351f63e85..e8546d422b 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -3413,7 +3413,6 @@ LLSD LLAppViewer::getViewerInfo() const info["PACKETS_LOST"] = LLViewerStats::instance().getRecording().getSum(LLStatViewer::PACKETS_LOST); info["PACKETS_IN"] = packets_in; info["PACKETS_PCT"] = 100.f*info["PACKETS_LOST"].asReal() / info["PACKETS_IN"].asReal(); - info["PACKETS_DROPPED"] = LLViewerStats::instance().getRecording().getSum(LLStatViewer::PACKETS_DROPPED); } if (mServerReleaseNotesURL.empty()) diff --git a/indra/newview/llviewerstats.cpp b/indra/newview/llviewerstats.cpp index 4f52ab644c..73aabf49d1 100644 --- a/indra/newview/llviewerstats.cpp +++ b/indra/newview/llviewerstats.cpp @@ -121,7 +121,6 @@ LLTrace::CountStatHandle<> FPS("FPS", "Frames rendered"), PACKETS_IN("Packets In", "Packets received"), PACKETS_LOST("packetsloststat", "Packets lost"), PACKETS_OUT("packetsoutstat", "Packets sent"), - PACKETS_DROPPED("packetsdropped", "Packets dropped"), TEXTURE_PACKETS("texturepacketsstat", "Texture data packets received"), CHAT_COUNT("chatcount", "Chat messages sent"), IM_COUNT("imcount", "IMs sent"), @@ -648,7 +647,6 @@ void send_viewer_stats(bool include_preferences) fail["send_packet"] = (S32) gMessageSystem->mSendPacketFailureCount; fail["dropped"] = (S32) gMessageSystem->mDroppedPackets; - fail["ring_dropped"] = (S32)gMessageSystem->mPacketRing.getNumDroppedPackets(); fail["resent"] = (S32) gMessageSystem->mResentPackets; fail["failed_resends"] = (S32) gMessageSystem->mFailedResendPackets; fail["off_circuit"] = (S32) gMessageSystem->mOffCircuitPackets; diff --git a/indra/newview/llviewerstats.h b/indra/newview/llviewerstats.h index 4cab2b48a5..8aed1c537e 100644 --- a/indra/newview/llviewerstats.h +++ b/indra/newview/llviewerstats.h @@ -119,7 +119,6 @@ extern LLTrace::CountStatHandle<> FPS, PACKETS_IN, PACKETS_LOST, PACKETS_OUT, - PACKETS_DROPPED, TEXTURE_PACKETS, CHAT_COUNT, IM_COUNT, diff --git a/indra/newview/llviewerthrottle.cpp b/indra/newview/llviewerthrottle.cpp index dce85bcb03..b0a00c29a4 100644 --- a/indra/newview/llviewerthrottle.cpp +++ b/indra/newview/llviewerthrottle.cpp @@ -304,7 +304,6 @@ void LLViewerThrottle::updateDynamicThrottle() } mUpdateTimer.reset(); - // Todo: account for dropped packets from LLPacketRing (or make the thing threaded) LLUnit mean_packets_lost = LLViewerStats::instance().getRecording().getMean(LLStatViewer::PACKETS_LOST_PERCENT); if (mean_packets_lost > TIGHTEN_THROTTLE_THRESHOLD) { diff --git a/indra/newview/llworld.cpp b/indra/newview/llworld.cpp index a1eed0340b..899733ccc3 100644 --- a/indra/newview/llworld.cpp +++ b/indra/newview/llworld.cpp @@ -795,7 +795,6 @@ void LLWorld::updateNetStats() S32 packets_in = gMessageSystem->mPacketsIn - mLastPacketsIn; S32 packets_out = gMessageSystem->mPacketsOut - mLastPacketsOut; S32 packets_lost = gMessageSystem->mDroppedPackets - mLastPacketsLost; - S32 ring_packets_dropped = gMessageSystem->mPacketRing.getNumDroppedPackets(); F64Bits actual_in_bits(gMessageSystem->mPacketRing.getAndResetActualInBits()); F64Bits actual_out_bits(gMessageSystem->mPacketRing.getAndResetActualOutBits()); @@ -806,7 +805,6 @@ void LLWorld::updateNetStats() add(LLStatViewer::PACKETS_IN, packets_in); add(LLStatViewer::PACKETS_OUT, packets_out); add(LLStatViewer::PACKETS_LOST, packets_lost); - add(LLStatViewer::PACKETS_DROPPED, ring_packets_dropped); F32 total_packets_in = (F32)LLViewerStats::instance().getRecording().getSum(LLStatViewer::PACKETS_IN); if (total_packets_in > 0.f) -- cgit v1.3 From c99e3167ed8549bc13d7df03b1e12dc15b0a080f Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Thu, 13 Mar 2025 23:55:38 +0200 Subject: #3644 Adjust throttle based of how busy buffer is --- indra/llmessage/llpacketring.cpp | 6 ++++++ indra/llmessage/llpacketring.h | 1 + indra/llmessage/message.h | 4 +--- indra/newview/llappviewer.cpp | 1 + indra/newview/llviewerthrottle.cpp | 14 +++++++++++--- indra/newview/llviewerthrottle.h | 3 +++ 6 files changed, 23 insertions(+), 6 deletions(-) (limited to 'indra/llmessage/llpacketring.cpp') diff --git a/indra/llmessage/llpacketring.cpp b/indra/llmessage/llpacketring.cpp index da3c502e9d..eb6650c6c5 100644 --- a/indra/llmessage/llpacketring.cpp +++ b/indra/llmessage/llpacketring.cpp @@ -344,6 +344,12 @@ bool LLPacketRing::expandRing() return true; } +F32 LLPacketRing::getBufferLoadRate() const +{ + // goes up to MAX_BUFFER_RING_SIZE + return (F32)mNumBufferedPackets / (F32)DEFAULT_BUFFER_RING_SIZE; +} + void LLPacketRing::dumpPacketRingStats() { mNumDroppedPacketsTotal += mNumDroppedPackets; diff --git a/indra/llmessage/llpacketring.h b/indra/llmessage/llpacketring.h index 237efc12e0..572dcbd271 100644 --- a/indra/llmessage/llpacketring.h +++ b/indra/llmessage/llpacketring.h @@ -64,6 +64,7 @@ public: S32 getNumBufferedBytes() const { return mNumBufferedBytes; } S32 getNumDroppedPackets() const { return mNumDroppedPacketsTotal + mNumDroppedPackets; } + F32 getBufferLoadRate() const; // from 0 to 4 (0 - empty, 1 - default size is full) void dumpPacketRingStats(); protected: // returns 'true' if we should intentionally drop a packet diff --git a/indra/llmessage/message.h b/indra/llmessage/message.h index 1844d5e7cd..30945cac51 100644 --- a/indra/llmessage/message.h +++ b/indra/llmessage/message.h @@ -538,7 +538,6 @@ public: //void buildMessage(); - S32 zeroCode(U8 **data, S32 *data_size); S32 zeroCodeExpand(U8 **data, S32 *data_size); S32 zeroCodeAdjustCurrentSendTotal(); @@ -755,6 +754,7 @@ public: S32 getReceiveBytes() const; S32 getUnackedListSize() const { return mUnackedListSize; } + F32 getBufferLoadRate() const { return mPacketRing.getBufferLoadRate(); } //const char* getCurrentSMessageName() const { return mCurrentSMessageName; } //const char* getCurrentSBlockName() const { return mCurrentSBlockName; } @@ -842,12 +842,10 @@ private: LLUUID mSessionID; void addTemplate(LLMessageTemplate *templatep); - bool decodeTemplate( const U8* buffer, S32 buffer_size, LLMessageTemplate** msg_template ); void logMsgFromInvalidCircuit( const LLHost& sender, bool recv_reliable ); void logTrustedMsgFromUntrustedCircuit( const LLHost& sender ); void logValidMsg(LLCircuitData *cdp, const LLHost& sender, bool recv_reliable, bool recv_resent, bool recv_acks ); - void logRanOffEndOfPacket( const LLHost& sender ); class LLMessageCountInfo { diff --git a/indra/newview/llappviewer.cpp b/indra/newview/llappviewer.cpp index a05c2376a0..4cf651de33 100644 --- a/indra/newview/llappviewer.cpp +++ b/indra/newview/llappviewer.cpp @@ -5443,6 +5443,7 @@ void LLAppViewer::idleNetwork() // Retransmit unacknowledged packets. gXferManager->retransmitUnackedPackets(); gAssetStorage->checkForTimeouts(); + gViewerThrottle.setBufferLoadRate(gMessageSystem->getBufferLoadRate()); gViewerThrottle.updateDynamicThrottle(); // Check that the circuit between the viewer and the agent's current diff --git a/indra/newview/llviewerthrottle.cpp b/indra/newview/llviewerthrottle.cpp index b0a00c29a4..8d935e4243 100644 --- a/indra/newview/llviewerthrottle.cpp +++ b/indra/newview/llviewerthrottle.cpp @@ -48,6 +48,8 @@ const F32 MIN_FRACTIONAL = 0.2f; const F32 MIN_BANDWIDTH = 50.f; const F32 MAX_BANDWIDTH = 6000.f; const F32 STEP_FRACTIONAL = 0.1f; +const F32 HIGH_BUFFER_LOAD_TRESHOLD = 1.f; +const F32 LOW_BUFFER_LOAD_TRESHOLD = 0.8f; const LLUnit TIGHTEN_THROTTLE_THRESHOLD(3.0f); // packet loss % per s const LLUnit EASE_THROTTLE_THRESHOLD(0.5f); // packet loss % per s const F32 DYNAMIC_UPDATE_DURATION = 5.0f; // seconds @@ -146,7 +148,7 @@ LLViewerThrottleGroup LLViewerThrottleGroup::operator-(const LLViewerThrottleGro void LLViewerThrottleGroup::sendToSim() const { - LL_INFOS() << "Sending throttle settings, total BW " << mThrottleTotal << LL_ENDL; + LL_DEBUGS("Throttle") << "Sending throttle settings, total BW " << mThrottleTotal << LL_ENDL; LLMessageSystem* msg = gMessageSystem; msg->newMessageFast(_PREHASH_AgentThrottle); @@ -305,7 +307,10 @@ void LLViewerThrottle::updateDynamicThrottle() mUpdateTimer.reset(); LLUnit mean_packets_lost = LLViewerStats::instance().getRecording().getMean(LLStatViewer::PACKETS_LOST_PERCENT); - if (mean_packets_lost > TIGHTEN_THROTTLE_THRESHOLD) + if ( + mean_packets_lost > TIGHTEN_THROTTLE_THRESHOLD // already losing packets + || mBufferLoadRate >= HIGH_BUFFER_LOAD_TRESHOLD // let viewer sort through the backlog before it starts dropping packets + ) { if (mThrottleFrac <= MIN_FRACTIONAL || mCurrentBandwidth / 1024.0f <= MIN_BANDWIDTH) { @@ -318,7 +323,8 @@ void LLViewerThrottle::updateDynamicThrottle() mCurrent.sendToSim(); LL_INFOS() << "Tightening network throttle to " << mCurrentBandwidth << LL_ENDL; } - else if (mean_packets_lost <= EASE_THROTTLE_THRESHOLD) + else if (mean_packets_lost <= EASE_THROTTLE_THRESHOLD + && mBufferLoadRate < LOW_BUFFER_LOAD_TRESHOLD) { if (mThrottleFrac >= MAX_FRACTIONAL || mCurrentBandwidth / 1024.0f >= MAX_BANDWIDTH) { @@ -331,4 +337,6 @@ void LLViewerThrottle::updateDynamicThrottle() mCurrent.sendToSim(); LL_INFOS() << "Easing network throttle to " << mCurrentBandwidth << LL_ENDL; } + + mBufferLoadRate = 0; } diff --git a/indra/newview/llviewerthrottle.h b/indra/newview/llviewerthrottle.h index 28a24d04fc..9973c88549 100644 --- a/indra/newview/llviewerthrottle.h +++ b/indra/newview/llviewerthrottle.h @@ -70,12 +70,15 @@ public: void updateDynamicThrottle(); void resetDynamicThrottle(); + void setBufferLoadRate(F32 rate) { mBufferLoadRate = llmax(mBufferLoadRate, rate); } + LLViewerThrottleGroup getThrottleGroup(const F32 bandwidth_kbps); static const std::string sNames[TC_EOF]; protected: F32 mMaxBandwidth; F32 mCurrentBandwidth; + F32 mBufferLoadRate = 0; LLViewerThrottleGroup mCurrent; -- cgit v1.3 From 8665cd3750767cf47246d1b89b8a668bc4dd4138 Mon Sep 17 00:00:00 2001 From: Andrey Kleshchev Date: Fri, 25 Apr 2025 21:31:43 +0300 Subject: #3986 Packet ring crash in memcpy packet_size was negative --- indra/llmessage/llpacketring.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'indra/llmessage/llpacketring.cpp') diff --git a/indra/llmessage/llpacketring.cpp b/indra/llmessage/llpacketring.cpp index eb6650c6c5..b8284334ea 100644 --- a/indra/llmessage/llpacketring.cpp +++ b/indra/llmessage/llpacketring.cpp @@ -209,8 +209,14 @@ S32 LLPacketRing::receiveOrDropBufferedPacket(char *datap, bool drop) if (!drop) { - assert(packet_size > 0); - memcpy(datap, packet->getData(), packet_size); + if (packet_size > 0) + { + memcpy(datap, packet->getData(), packet_size); + } + else + { + assert(false); + } } else { -- cgit v1.3