Application limited time in packet metadata

Summary:
Track the total (cumulative) amount of time that the connection has been application limited and store this information in `OutstandingPacketMetadata`. If this value is the same for two `OutstandingPacketMetadata` then we know that the transport did not become application limited between when those two packets were sent (or, less likely, the transport was application limited for less than one microsecond given the microsecond resolution of the timestamp).

We store the amount of time spent application limited instead of a count of the number of application limited events because the implications of being application limited are time dependent.

Tests show that we need to be able to inject a mockable clock. That's been an issue for some time; will work on in a subsequent diff.

Differential Revision: D41714879

fbshipit-source-id: 9fd4fe321d85639dc9fb5c2cd51713c481cbeb22

quic/api/QuicTransportBase.cpp

@@ -3160,9 +3160,10 @@ void QuicTransportBase::writeSocketData() {
         conn_->outstandings.numOutstanding();
 
     // if we're starting to write from app limited, notify observers
-    if (conn_->waitingForAppData && conn_->congestionController) {
+    if (conn_->appLimitedTracker.isAppLimited() &&
+        conn_->congestionController) {
+      conn_->appLimitedTracker.setNotAppLimited();
       notifyStartWritingFromAppRateLimited();
-      conn_->waitingForAppData = false;
     }
     writeData();
     if (closeState_ != CloseState::CLOSED) {
@@ -3241,8 +3242,8 @@ void QuicTransportBase::writeSocketData() {
         if (transportReadyNotified_ && connCallback_) {
           connCallback_->onAppRateLimited();
         }
+        conn_->appLimitedTracker.setAppLimited();
         notifyAppRateLimited();
-        conn_->waitingForAppData = true;
       }
     }
   }

quic/api/QuicTransportFunctions.cpp

@@ -884,7 +884,8 @@ void updateConnection(
       conn.outstandings.numOutstanding() + 1,
       conn.lossState,
       conn.writeCount,
-      std::move(detailsPerStream));
+      std::move(detailsPerStream),
+      conn.appLimitedTracker.getTotalAppLimitedTime());
 
   if (isD6DProbe) {
     ++conn.d6d.outstandingProbes;

quic/api/test/QuicTransportFunctionsTest.cpp

@@ -1520,6 +1520,102 @@ TEST_F(QuicTransportFunctionsTest, TestUpdateConnectionWithBytesStats) {
           ->lastAckedPacketInfo->totalBytesAcked);
 }
 
+TEST_F(QuicTransportFunctionsTest, TestUpdateConnectionWithAppLimitedStats) {
+  auto conn = createConn();
+
+  auto stream = conn->streamManager->createNextBidirectionalStream().value();
+  auto packet = buildEmptyPacket(*conn, PacketNumberSpace::Handshake);
+  writeDataToQuicStream(
+      *stream, folly::IOBuf::copyBuffer("Im gonna cut your hair."), true);
+  WriteStreamFrame writeStreamFrame(stream->id, 0, 5, false);
+  packet.packet.frames.push_back(std::move(writeStreamFrame));
+
+  // connections should start off being app limited
+  EXPECT_TRUE(conn->appLimitedTracker.isAppLimited());
+
+  // mark ourselves as not app limited, verify that total app limited time > 0,
+  // verify that successive calls to getTotalAppLimitedTime yield same value
+  conn->appLimitedTracker.setNotAppLimited();
+  EXPECT_LT(0us, conn->appLimitedTracker.getTotalAppLimitedTime());
+  EXPECT_EQ(
+      conn->appLimitedTracker.getTotalAppLimitedTime(),
+      conn->appLimitedTracker.getTotalAppLimitedTime());
+
+  // record the packet as having been sent
+  updateConnection(
+      *conn,
+      folly::none,
+      packet.packet,
+      TimePoint(),
+      555,
+      500,
+      false /* isDSRPacket */);
+
+  // should have the current app limited time recorded in metadata
+  EXPECT_EQ(
+      conn->appLimitedTracker.getTotalAppLimitedTime(),
+      getFirstOutstandingPacket(*conn, PacketNumberSpace::Handshake)
+          ->metadata.totalAppLimitedTimeUsecs);
+}
+
+TEST_F(
+    QuicTransportFunctionsTest,
+    TestUpdateConnectionWithAppLimitedStats_MultipleTransitions) {
+  auto conn = createConn();
+
+  auto stream = conn->streamManager->createNextBidirectionalStream().value();
+  auto packet = buildEmptyPacket(*conn, PacketNumberSpace::Handshake);
+  writeDataToQuicStream(
+      *stream, folly::IOBuf::copyBuffer("Im gonna cut your hair."), true);
+  WriteStreamFrame writeStreamFrame(stream->id, 0, 5, false);
+  packet.packet.frames.push_back(std::move(writeStreamFrame));
+
+  // connections should start off being app limited
+  EXPECT_TRUE(conn->appLimitedTracker.isAppLimited());
+  {
+    // successive calls should yield different measurements
+    const auto time1 = conn->appLimitedTracker.getTotalAppLimitedTime();
+    std::this_thread::sleep_for(10ms);
+    const auto time2 = conn->appLimitedTracker.getTotalAppLimitedTime();
+    EXPECT_NE(time1, time2);
+  }
+
+  // mark ourselves as not app limited, verify that total app limited time > 0,
+  // verify that successive calls to getTotalAppLimitedTime yield same value
+  conn->appLimitedTracker.setNotAppLimited();
+  EXPECT_LT(0us, conn->appLimitedTracker.getTotalAppLimitedTime());
+  EXPECT_EQ(
+      conn->appLimitedTracker.getTotalAppLimitedTime(),
+      conn->appLimitedTracker.getTotalAppLimitedTime());
+  const auto appLimitedTime1 = conn->appLimitedTracker.getTotalAppLimitedTime();
+
+  // become app limited for at least 10ms, then repeat the above
+  conn->appLimitedTracker.setAppLimited();
+  std::this_thread::sleep_for(10ms);
+  conn->appLimitedTracker.setNotAppLimited();
+  EXPECT_LE(
+      appLimitedTime1 + 10ms, conn->appLimitedTracker.getTotalAppLimitedTime());
+  EXPECT_EQ(
+      conn->appLimitedTracker.getTotalAppLimitedTime(),
+      conn->appLimitedTracker.getTotalAppLimitedTime());
+
+  // record the packet as having been sent
+  updateConnection(
+      *conn,
+      folly::none,
+      packet.packet,
+      TimePoint(),
+      555,
+      500,
+      false /* isDSRPacket */);
+
+  // should have the current app limited time recorded in metadata
+  EXPECT_EQ(
+      conn->appLimitedTracker.getTotalAppLimitedTime(),
+      getFirstOutstandingPacket(*conn, PacketNumberSpace::Handshake)
+          ->metadata.totalAppLimitedTimeUsecs);
+}
+
 TEST_F(QuicTransportFunctionsTest, TestUpdateConnectionWithCloneResult) {
   auto conn = createConn();
   conn->qLogger = std::make_shared<quic::FileQLogger>(VantagePoint::Client);

quic/api/test/QuicTypedTransportTest.cpp

@@ -736,6 +736,163 @@ TYPED_TEST(
   this->destroyTransport();
 }
 
+/**
+ * Verify app limited time tracking and annotation.
+ */
+TYPED_TEST(QuicTypedTransportAfterStartTest, TotalAppLimitedTime) {
+  // ACK outstanding packets so that we can switch out the congestion control
+  this->ackAllOutstandingPackets();
+  EXPECT_THAT(this->getConn().outstandings.packets, IsEmpty());
+
+  // install StaticCwndCongestionController
+  const auto cwndInBytes = 7000;
+  this->getNonConstConn().congestionController =
+      std::make_unique<StaticCwndCongestionController>(
+          StaticCwndCongestionController::CwndInBytes(cwndInBytes));
+
+  // install PacketProcessor
+  auto mockPacketProcessor = std::make_unique<MockPacketProcessor>();
+  auto rawPacketProcessor = mockPacketProcessor.get();
+  this->getNonConstConn().packetProcessors.push_back(
+      std::move(mockPacketProcessor));
+
+  auto streamId = this->getTransport()->createBidirectionalStream().value();
+
+  // write 1700 bytes to stream to generate two packets back to back
+  // both packets should have the same app limited time
+  auto firstPacketTotalAppLimitedTimeUsecs = 0us;
+  {
+    EXPECT_CALL(*rawPacketProcessor, onPacketSent(_))
+        .Times(2)
+        .WillOnce(Invoke([&](auto outstandingPacket) {
+          EXPECT_EQ(4, outstandingPacket.metadata.totalPacketsSent);
+          EXPECT_EQ(1, outstandingPacket.metadata.packetsInflight);
+          EXPECT_EQ(3, outstandingPacket.metadata.writeCount);
+          EXPECT_NE(0us, outstandingPacket.metadata.totalAppLimitedTimeUsecs);
+          firstPacketTotalAppLimitedTimeUsecs =
+              outstandingPacket.metadata.totalAppLimitedTimeUsecs;
+        }))
+        .WillOnce(Invoke([&](auto outstandingPacket) {
+          EXPECT_EQ(5, outstandingPacket.metadata.totalPacketsSent);
+          EXPECT_EQ(2, outstandingPacket.metadata.packetsInflight);
+          EXPECT_EQ(3, outstandingPacket.metadata.writeCount);
+          EXPECT_EQ(
+              firstPacketTotalAppLimitedTimeUsecs,
+              outstandingPacket.metadata.totalAppLimitedTimeUsecs);
+        }));
+
+    const auto bufLength = 1700;
+    auto buf = buildRandomInputData(bufLength);
+    this->getTransport()->writeChain(streamId, std::move(buf), false);
+    const auto maybeWrittenPackets1 = this->loopForWrites();
+
+    // should have sent two packets
+    ASSERT_TRUE(maybeWrittenPackets1.has_value());
+    quic::PacketNum firstPacketNum = maybeWrittenPackets1->start;
+    quic::PacketNum lastPacketNum = maybeWrittenPackets1->end;
+    EXPECT_EQ(2, lastPacketNum - firstPacketNum + 1);
+  }
+
+  // now we're going to be application limited for 10ms (or more)
+  std::this_thread::sleep_for(10ms);
+
+  // write 10000 bytes to stream to generate multiple packets back to back
+  // not all will be sent at once because our CWND is only 7000 bytes, and we
+  // already used 1700 bytes+ in previous send
+  //
+  // when (eventually) sent, all packets should have
+  //   - the same app limited time
+  //   - app limited time >= 10ms + firstPacketTotalAppLimitedTimeUsecs
+  auto thirdPacketTotalAppLimitedTimeUsecs = 0us;
+  {
+    EXPECT_CALL(*rawPacketProcessor, onPacketSent(_))
+        .Times(4)
+        .WillOnce(Invoke([&](auto outstandingPacket) {
+          EXPECT_EQ(4, outstandingPacket.metadata.writeCount);
+          EXPECT_LE(
+              firstPacketTotalAppLimitedTimeUsecs + 10ms,
+              outstandingPacket.metadata.totalAppLimitedTimeUsecs);
+          thirdPacketTotalAppLimitedTimeUsecs =
+              outstandingPacket.metadata.totalAppLimitedTimeUsecs;
+        }))
+        .WillRepeatedly(Invoke([&](auto outstandingPacket) {
+          EXPECT_EQ(
+              thirdPacketTotalAppLimitedTimeUsecs,
+              outstandingPacket.metadata.totalAppLimitedTimeUsecs);
+        }));
+
+    const auto bufLength = 10000;
+    auto buf = buildRandomInputData(bufLength);
+    this->getTransport()->writeChain(streamId, std::move(buf), false);
+    const auto maybeWrittenPackets = this->loopForWrites();
+
+    // should have sent five packets
+    ASSERT_TRUE(maybeWrittenPackets.has_value());
+    quic::PacketNum firstPacketNum = maybeWrittenPackets->start;
+    quic::PacketNum lastPacketNum = maybeWrittenPackets->end;
+    EXPECT_EQ(4, lastPacketNum - firstPacketNum + 1);
+  }
+
+  // deliver an ACK for all of the outstanding packets
+  this->ackAllOutstandingPackets();
+
+  // finish sending the rest of the packets
+  // they should have the same app limited time as packet #3
+  {
+    EXPECT_CALL(*rawPacketProcessor, onPacketSent(_))
+        .Times(3)
+        .WillRepeatedly(Invoke([&](auto outstandingPacket) {
+          EXPECT_EQ(
+              thirdPacketTotalAppLimitedTimeUsecs,
+              outstandingPacket.metadata.totalAppLimitedTimeUsecs);
+        }));
+
+    const auto maybeWrittenPackets = this->loopForWrites();
+    ASSERT_TRUE(maybeWrittenPackets.has_value());
+    quic::PacketNum firstPacketNum = maybeWrittenPackets->start;
+    quic::PacketNum lastPacketNum = maybeWrittenPackets->end;
+    EXPECT_EQ(3, lastPacketNum - firstPacketNum + 1);
+  }
+
+  // deliver an ACK for all of the outstanding packets
+  this->ackAllOutstandingPackets();
+
+  // now we're going to be application limited again for 10ms (or more)
+  std::this_thread::sleep_for(10ms);
+
+  // finally, write 1700 bytes again, and verify we see a new app limited time
+  {
+    auto penultimatePacketTotalAppLimitedTimeUsecs = 0us;
+    EXPECT_CALL(*rawPacketProcessor, onPacketSent(_))
+        .Times(2)
+        .WillOnce(Invoke([&](auto outstandingPacket) {
+          EXPECT_LE(
+              thirdPacketTotalAppLimitedTimeUsecs + 10ms,
+              outstandingPacket.metadata.totalAppLimitedTimeUsecs);
+          penultimatePacketTotalAppLimitedTimeUsecs =
+              outstandingPacket.metadata.totalAppLimitedTimeUsecs;
+        }))
+        .WillOnce(Invoke([&](auto outstandingPacket) {
+          EXPECT_EQ(
+              penultimatePacketTotalAppLimitedTimeUsecs,
+              outstandingPacket.metadata.totalAppLimitedTimeUsecs);
+        }));
+
+    const auto bufLength = 1700;
+    auto buf = buildRandomInputData(bufLength);
+    this->getTransport()->writeChain(streamId, std::move(buf), false);
+    const auto maybeWrittenPackets1 = this->loopForWrites();
+
+    // should have sent two packets
+    ASSERT_TRUE(maybeWrittenPackets1.has_value());
+    quic::PacketNum firstPacketNum = maybeWrittenPackets1->start;
+    quic::PacketNum lastPacketNum = maybeWrittenPackets1->end;
+    EXPECT_EQ(2, lastPacketNum - firstPacketNum + 1);
+  }
+
+  this->destroyTransport();
+}
+
 TYPED_TEST(
     QuicTypedTransportAfterStartTest,
     StreamAckedIntervalsDeliveryCallbacks) {

quic/state/OutstandingPacket.h

@@ -10,6 +10,7 @@
 #include <quic/codec/Types.h>
 #include <quic/state/LossState.h>
 #include <quic/state/PacketEvent.h>
+#include <chrono>
 
 namespace quic {
 
@@ -104,6 +105,10 @@ struct OutstandingPacketMetadata {
   // Details about each stream with frames in this packet
   DetailsPerStream detailsPerStream;
 
+  // Total time spent app limited on this connection including when this packet
+  // was sent.
+  std::chrono::microseconds totalAppLimitedTimeUsecs{0};
+
   OutstandingPacketMetadata(
       TimePoint timeIn,
       uint32_t encodedSizeIn,
@@ -116,7 +121,8 @@ struct OutstandingPacketMetadata {
       uint64_t packetsInflightIn,
       const LossState& lossStateIn,
       uint64_t writeCount,
-      DetailsPerStream detailsPerStream)
+      DetailsPerStream detailsPerStream,
+      std::chrono::microseconds totalAppLimitedTimeUsecsIn = 0us)
       : time(timeIn),
         encodedSize(encodedSizeIn),
         encodedBodySize(encodedBodySizeIn),
@@ -129,7 +135,8 @@ struct OutstandingPacketMetadata {
         totalPacketsSent(lossStateIn.totalPacketsSent),
         totalAckElicitingPacketsSent(lossStateIn.totalAckElicitingPacketsSent),
         writeCount(writeCount),
-        detailsPerStream(std::move(detailsPerStream)) {}
+        detailsPerStream(std::move(detailsPerStream)),
+        totalAppLimitedTimeUsecs(totalAppLimitedTimeUsecsIn) {}
 };
 
 // Data structure to represent outstanding retransmittable packets
@@ -206,7 +213,8 @@ struct OutstandingPacket {
       uint64_t packetsInflightIn,
       const LossState& lossStateIn,
       uint64_t writeCount,
-      Metadata::DetailsPerStream detailsPerStream)
+      Metadata::DetailsPerStream detailsPerStream,
+      std::chrono::microseconds totalAppLimitedTimeUsecs = 0us)
       : packet(std::move(packetIn)),
         metadata(OutstandingPacketMetadata(
             timeIn,
@@ -220,7 +228,8 @@ struct OutstandingPacket {
             packetsInflightIn,
             lossStateIn,
             writeCount,
-            std::move(detailsPerStream))) {}
+            std::move(detailsPerStream),
+            totalAppLimitedTimeUsecs)) {}
 
   OutstandingPacket(
       RegularQuicWritePacket packetIn,
@@ -235,7 +244,8 @@ struct OutstandingPacket {
       uint64_t packetsInflightIn,
       const LossState& lossStateIn,
       uint64_t writeCount,
-      Metadata::DetailsPerStream detailsPerStream)
+      Metadata::DetailsPerStream detailsPerStream,
+      std::chrono::microseconds totalAppLimitedTimeUsecs = 0us)
       : packet(std::move(packetIn)),
         metadata(OutstandingPacketMetadata(
             timeIn,
@@ -249,6 +259,7 @@ struct OutstandingPacket {
             packetsInflightIn,
             lossStateIn,
             writeCount,
-            std::move(detailsPerStream))) {}
+            std::move(detailsPerStream),
+            totalAppLimitedTimeUsecs)) {}
 };
 } // namespace quic