[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [torbrowser/master] Bug 8470: Increase pipeline randomization.



commit f520a5037c96de05f208552158802fa4840055af
Author: Mike Perry <mikeperry-git@xxxxxxxxxx>
Date:   Tue Apr 2 16:02:56 2013 -0700

    Bug 8470: Increase pipeline randomization.
---
 build-scripts/config/pound_tor.js                  |    3 +
 ...ize-HTTP-request-order-and-pipeline-depth.patch |  708 +++++++++++++++++---
 2 files changed, 623 insertions(+), 88 deletions(-)

diff --git a/build-scripts/config/pound_tor.js b/build-scripts/config/pound_tor.js
index ab98b46..db382a6 100644
--- a/build-scripts/config/pound_tor.js
+++ b/build-scripts/config/pound_tor.js
@@ -97,6 +97,9 @@ pref("network.http.proxy.pipelining", true);
 pref("security.ssl.enable_false_start", true);
 pref("network.http.keep-alive.timeout", 20);
 pref("network.http.connection-retry-timeout", 0);
+pref("network.http.max-persistent-connections-per-proxy", 256);
+// Hacked pref: Now means "Attempt to pipeline at least this many requests together"
+pref("network.http.pipelining.max-optimistic-requests", 3);
 
 // Extension support
 pref("extensions.autoDisableScopes", 0);
diff --git a/src/current-patches/firefox/0017-Randomize-HTTP-request-order-and-pipeline-depth.patch b/src/current-patches/firefox/0017-Randomize-HTTP-request-order-and-pipeline-depth.patch
index ddfb184..a5d2957 100644
--- a/src/current-patches/firefox/0017-Randomize-HTTP-request-order-and-pipeline-depth.patch
+++ b/src/current-patches/firefox/0017-Randomize-HTTP-request-order-and-pipeline-depth.patch
@@ -1,7 +1,7 @@
-From cd87b7f64f035f67ec883c1b1ed4746454892781 Mon Sep 17 00:00:00 2001
+From a88adc5c3000c3e2a18b1065da77dd83d2b3ae7b Mon Sep 17 00:00:00 2001
 From: Mike Perry <mikeperry-git@xxxxxxxxxxxxxx>
 Date: Tue, 4 Dec 2012 17:38:51 -0800
-Subject: [PATCH 17/27] Randomize HTTP request order and pipeline depth.
+Subject: [PATCH 17/28] Randomize HTTP request order and pipeline depth.
 
 This is an experimental defense against
 http://lorre.uni.lu/~andriy/papers/acmccs-wpes11-fingerprinting.pdf
@@ -13,18 +13,21 @@ This defense has been improved since that blog post to additionally randomize
 the order and concurrency of non-pipelined HTTP requests.
 
 This patch is also different from the 10.x ESR patch, as the pipelining
-code has changed. We may want to set network.http.pipelining.aggressive to get
-similar behavior...
+code has changed significantly.
 
-The good news is we now randomize SPDY request order as well as pipeline
-request order (though SPDY is still disabled by default in TBB).
+This patch may have some minor impact on SPDY request order, but the SPDY
+implementation has not been altered directly. It has several stream queues
+that may also benefit from reordering.
 ---
- netwerk/protocol/http/nsHttpConnectionMgr.cpp |   67 +++++++++++++++++++++++--
- netwerk/protocol/http/nsHttpConnectionMgr.h   |    3 +
- 2 files changed, 65 insertions(+), 5 deletions(-)
+ netwerk/protocol/http/nsHttpConnectionMgr.cpp |  288 +++++++++++++++++--------
+ netwerk/protocol/http/nsHttpConnectionMgr.h   |   15 +-
+ netwerk/protocol/http/nsHttpHandler.h         |    2 +
+ netwerk/protocol/http/nsHttpPipeline.cpp      |   62 +++++-
+ netwerk/protocol/http/nsHttpPipeline.h        |    3 +
+ 5 files changed, 272 insertions(+), 98 deletions(-)
 
 diff --git a/netwerk/protocol/http/nsHttpConnectionMgr.cpp b/netwerk/protocol/http/nsHttpConnectionMgr.cpp
-index 133c301..59d03c0 100644
+index 133c301..872d505 100644
 --- a/netwerk/protocol/http/nsHttpConnectionMgr.cpp
 +++ b/netwerk/protocol/http/nsHttpConnectionMgr.cpp
 @@ -20,6 +20,8 @@
@@ -36,108 +39,465 @@ index 133c301..59d03c0 100644
  using namespace mozilla;
  using namespace mozilla::net;
  
-@@ -39,15 +41,46 @@ InsertTransactionSorted(nsTArray<nsHttpTransaction*> &pendingQ, nsHttpTransactio
+@@ -39,15 +41,26 @@ InsertTransactionSorted(nsTArray<nsHttpTransaction*> &pendingQ, nsHttpTransactio
      // insert into queue with smallest valued number first.  search in reverse
      // order under the assumption that many of the existing transactions will
      // have the same priority (usually 0).
 +    uint32_t len = pendingQ.Length();
-+    uint32_t begin = 0, end = len+1;
-+    int found_begin = 0;
  
 -    for (int32_t i=pendingQ.Length()-1; i>=0; --i) {
+-        nsHttpTransaction *t = pendingQ[i];
+-        if (trans->Priority() >= t->Priority()) {
+-            pendingQ.InsertElementAt(i+1, trans);
+-            return;
+-        }
 +    if (pendingQ.IsEmpty()) {
 +        pendingQ.InsertElementAt(0, trans);
 +        return;
-+    }
+     }
 +
-+// #define PRESERVE_PRIORITY_ORDER 
-+#ifdef PRESERVE_PRIORITY_ORDER
-+    // XXX: Untested
-+    for (uint32_t i=0; i < len; ++i) {
-         nsHttpTransaction *t = pendingQ[i];
--        if (trans->Priority() >= t->Priority()) {
--            pendingQ.InsertElementAt(i+1, trans);
--            return;
+     pendingQ.InsertElementAt(0, trans);
 +
-+        /* As soon as we see a priority >= us, our insertion
-+         * range starts there */
-+        if (!found_begin && t->Priority() >= trans->Priority()) {
-+            begin = i;
-+            found_begin = 1;
-+        }
-+        /* As soon as we see a priority > us, our insertion
-+         * range ends there */
-+        if (t->Priority() > trans->Priority()) {
-+            end = i;
-+            break;
++    // FIXME: Refactor into standalone helper (for nsHttpPipeline)
++    // Or at least simplify this function if this shuffle ends up
++    // being an improvement.
++    uint32_t i = 0;
++    for (i=0; i < len; ++i) {
++        uint32_t ridx = rand() % len;
++
++        nsHttpTransaction *tmp = pendingQ[i];
++        pendingQ[i] = pendingQ[ridx];
++        pendingQ[ridx] = tmp;
++    }
+ }
+ 
+ //-----------------------------------------------------------------------------
+@@ -919,6 +932,8 @@ nsHttpConnectionMgr::ProcessPendingQForEntry(nsConnectionEntry *ent)
+     nsHttpTransaction *trans;
+     nsresult rv;
+     bool dispatchedSuccessfully = false;
++    int dispatchCount = 0;
++    int total = count;
+ 
+     // iterate the pending list until one is dispatched successfully. Keep
+     // iterating afterwards only until a transaction fails to dispatch.
+@@ -953,16 +968,29 @@ nsHttpConnectionMgr::ProcessPendingQForEntry(nsConnectionEntry *ent)
+             dispatchedSuccessfully = true;
+             count = ent->mPendingQ.Length();
+             --i;
++            dispatchCount++;
+             continue;
          }
+ 
+-        if (dispatchedSuccessfully)
+-            return true;
++        // We want to keep walking the dispatch table to ensure requests
++        // get combined properly.
++        //if (dispatchedSuccessfully) {
++        //    return true;
++        //}
+ 
+         NS_ABORT_IF_FALSE(count == ((int32_t) ent->mPendingQ.Length()),
+                           "something mutated pending queue from "
+                           "GetConnection()");
      }
--    pendingQ.InsertElementAt(0, trans);
 +
-+    // XXX Verify that begin..end are all == trans->Priority()
++#ifdef WTF_DEBUG
++    if (dispatchedSuccessfully) {
++        fprintf(stderr, "WTF-queue: Dispatched %d/%d pending transactions for %s\n",
++                dispatchCount, total, ent->mConnInfo->Host());
++        return true;
++    }
 +#endif
 +
-+    // Choose random destination begin..end
-+    uint32_t count = end - begin;
-+    if (count == 0) count = 1;
-+
-+    // FIXME: rand() is not crypto-secure.. but meh, this code will probably
-+    // change like 2 dozen more times before merge, and rand() is probably 
-+    // good enough for our purposes anyways.
-+    pendingQ.InsertElementAt(begin + (rand()%count), trans);
+     return false;
  }
  
- //-----------------------------------------------------------------------------
-@@ -68,6 +101,12 @@ nsHttpConnectionMgr::nsHttpConnectionMgr()
-     mCT.Init();
-     mAlternateProtocolHash.Init(16);
-     mSpdyPreferredHash.Init();
-+
-+    nsresult rv;
-+    mRandomGenerator = do_GetService("@mozilla.org/security/random-generator;1", &rv);
-+    if (NS_FAILED(rv)) {
-+        mRandomGenerator = nullptr;
-+    }
+@@ -1263,7 +1291,7 @@ nsHttpConnectionMgr::MakeNewConnection(nsConnectionEntry *ent,
  }
  
- nsHttpConnectionMgr::~nsHttpConnectionMgr()
-@@ -1120,6 +1159,19 @@ nsHttpConnectionMgr::AtActiveConnectionLimit(nsConnectionEntry *ent, uint8_t cap
+ bool
+-nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent,
++nsHttpConnectionMgr::AddToBestPipeline(nsConnectionEntry *ent,
+                                            nsHttpTransaction *trans,
+                                            nsHttpTransaction::Classifier classification,
+                                            uint16_t depthLimit)
+@@ -1300,40 +1328,92 @@ nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent,
+     if (maxdepth < 2)
+         return false;
+ 
+-    nsAHttpTransaction *activeTrans;
++    // Find out how many requests of this class we have
++    uint32_t sameClass = 0;
++    uint32_t allClasses = ent->mPendingQ.Length();
++    for (uint32_t i = 0; i < allClasses; ++i) {
++        if (trans != ent->mPendingQ[i] &&
++            classification == ent->mPendingQ[i]->Classification()) {
++            sameClass++;
++        }
++    }
+ 
++    nsAHttpTransaction *activeTrans;
++    nsHttpPipeline *pipeline;
+     nsHttpConnection *bestConn = nullptr;
+     uint32_t activeCount = ent->mActiveConns.Length();
+-    uint32_t bestConnLength = 0;
+-    uint32_t connLength;
++    uint32_t pipelineDepth;
++    uint32_t requestLen;
++    uint32_t totalDepth = 0;
++
++    // Now, try to find the best pipeline
++    nsTArray<nsHttpConnection *> validConns;
++    nsTArray<nsHttpConnection *> betterConns;
++    nsTArray<nsHttpConnection *> bestConns;
++    uint32_t numPipelines = 0;
+ 
+     for (uint32_t i = 0; i < activeCount; ++i) {
+         nsHttpConnection *conn = ent->mActiveConns[i];
+-        if (!conn->SupportsPipelining())
+-            continue;
  
-     LOG(("   connection count = %d, limit %d\n", totalCount, maxPersistConns));
+-        if (conn->Classification() != classification)
++        if (!conn->SupportsPipelining())
+             continue;
  
-+    // Fuzz maxConns for website fingerprinting attack
-+    // We create a range of maxConns/5 up to 6*maxConns/5 
-+    // because this function is called repeatedly, and we'll
-+    // end up converging on the high side of concurrent connections
-+    // after a short while. 
-+    PRUint8 *bytes = nullptr;
-+    nsresult rv = mRandomGenerator->GenerateRandomBytes(1, &bytes);
-+    NS_ENSURE_SUCCESS(rv, rv);
+         activeTrans = conn->Transaction();
 +
-+    bytes[0] = bytes[0] % (maxPersistConns + 1);
-+    maxPersistConns = (maxPersistConns/5) + bytes[0];
-+    NS_Free(bytes);
+         if (!activeTrans ||
+             activeTrans->IsDone() ||
+             NS_FAILED(activeTrans->Status()))
+             continue;
+ 
+-        connLength = activeTrans->PipelineDepth();
++        pipeline = activeTrans->QueryPipeline();
++        if (!pipeline)
++            continue;
++
++        numPipelines++;
 +
-     // use >= just to be safe
-     bool result = (totalCount >= maxPersistConns);
-     LOG(("  result: %s", result ? "true" : "false"));
-@@ -1297,6 +1349,11 @@ nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent,
++        pipelineDepth = activeTrans->PipelineDepth();
++        requestLen = pipeline->RequestDepth();
  
-     maxdepth = PR_MIN(maxdepth, depthLimit);
+-        if (maxdepth <= connLength)
++        totalDepth += pipelineDepth;
++
++        // Only count in-flight requests towards maxdepth.
++        if (maxdepth <= (pipelineDepth - requestLen))
+             continue;
  
-+    if (maxdepth/2 > 1) { 
-+      // This is a crazy hack to randomize pipeline depth a bit more..
-+      maxdepth = 1 + maxdepth/2 + (rand() % (maxdepth/2));
+-        if (!bestConn || (connLength < bestConnLength)) {
+-            bestConn = conn;
+-            bestConnLength = connLength;
+-        }
+-    }
++        validConns.AppendElement(conn);
++
++        // Prefer a pipeline that either has at least two requests
++        // queued already, or for which we can add multiple requests
++        if (requestLen + allClasses < mMaxOptimisticPipelinedRequests)
++            continue;
+ 
+-    if (!bestConn)
++        betterConns.AppendElement(conn);
++
++        // Prefer a pipeline with the same classification if 
++        // our current classes will put it over the line
++        if (conn->Classification() != classification)
++            continue;
++        if (requestLen + sameClass < mMaxOptimisticPipelinedRequests)
++            continue;
++
++        bestConns.AppendElement(conn);
 +    }
 +
-     if (maxdepth < 2)
++    const char *type;
++    if (bestConns.Length()) {
++        type = "best";
++        bestConn = bestConns[rand()%bestConns.Length()];
++    } else if (betterConns.Length()) {
++        type = "better";
++        bestConn = betterConns[rand()%betterConns.Length()];
++    } else if (validConns.Length() && totalDepth == 0) {
++        // We only use valid conns if it's a last resort
++        // (No other requests are pending or in flight)
++        type = "valid";
++        bestConn = validConns[rand()%validConns.Length()];
++    } else {
          return false;
++    }
+ 
+     activeTrans = bestConn->Transaction();
+     nsresult rv = activeTrans->AddTransaction(trans);
+@@ -1343,6 +1423,14 @@ nsHttpConnectionMgr::AddToShortestPipeline(nsConnectionEntry *ent,
+     LOG(("   scheduling trans %p on pipeline at position %d\n",
+          trans, trans->PipelinePosition()));
+ 
++#ifdef WTF_DEBUG
++    pipeline = activeTrans->QueryPipeline();
++    fprintf(stderr, "WTF-depth: Added trans to %s of %d/%d/%d/%d pipelines. Request len %d/%d for %s\n",
++            type, bestConns.Length(), betterConns.Length(), validConns.Length(),
++            numPipelines, pipeline->RequestDepth(), activeTrans->PipelineDepth(),
++            ent->mConnInfo->Host());
++#endif
++
+     if ((ent->PipelineState() == PS_YELLOW) && (trans->PipelinePosition() > 1))
+         ent->SetYellowConnection(bestConn);
+     return true;
+@@ -1403,26 +1491,12 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent,
+     nsHttpTransaction::Classifier classification = trans->Classification();
+     uint8_t caps = trans->Caps();
+ 
++    bool allowNewPipelines = true;
++
+     // no keep-alive means no pipelines either
+     if (!(caps & NS_HTTP_ALLOW_KEEPALIVE))
+         caps = caps & ~NS_HTTP_ALLOW_PIPELINING;
+ 
+-    // 0 - If this should use spdy then dispatch it post haste.
+-    // 1 - If there is connection pressure then see if we can pipeline this on
+-    //     a connection of a matching type instead of using a new conn
+-    // 2 - If there is an idle connection, use it!
+-    // 3 - if class == reval or script and there is an open conn of that type
+-    //     then pipeline onto shortest pipeline of that class if limits allow
+-    // 4 - If we aren't up against our connection limit,
+-    //     then open a new one
+-    // 5 - Try a pipeline if we haven't already - this will be unusual because
+-    //     it implies a low connection pressure situation where
+-    //     MakeNewConnection() failed.. that is possible, but unlikely, due to
+-    //     global limits
+-    // 6 - no connection is available - queue it
+-
+-    bool attemptedOptimisticPipeline = !(caps & NS_HTTP_ALLOW_PIPELINING);
+-
+     // step 0
+     // look for existing spdy connection - that's always best because it is
+     // essentially pipelining without head of line blocking
+@@ -1436,20 +1510,27 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent,
+         }
+     }
+ 
+-    // step 1
+-    // If connection pressure, then we want to favor pipelining of any kind
+-    if (IsUnderPressure(ent, classification) && !attemptedOptimisticPipeline) {
+-        attemptedOptimisticPipeline = true;
+-        if (AddToShortestPipeline(ent, trans,
+-                                  classification,
+-                                  mMaxOptimisticPipelinedRequests)) {
+-            return NS_OK;
+-        }
++    // step 1: Try a pipeline
++    if (caps & NS_HTTP_ALLOW_PIPELINING &&
++        AddToBestPipeline(ent, trans, classification,
++                          mMaxPipelinedRequests)) {
++        return NS_OK;
+     }
+ 
+-    // step 2
+-    // consider an idle persistent connection
+-    if (caps & NS_HTTP_ALLOW_KEEPALIVE) {
++    // Step 2: Decide if we should forbid new pipeline creation.
++    //
++    // FIXME: We repurposed mMaxOptimisticPipelinedRequests here to mean:
++    // "Don't make a new pipeline until you have this many requests pending and 
++    // no potential connections to put them on". It might be nice to give this
++    // its own pref..
++    if (HasPipelines(ent) &&
++            ent->mPendingQ.Length() < mMaxOptimisticPipelinedRequests &&
++            trans->Classification() != nsAHttpTransaction::CLASS_SOLO &&
++            caps & NS_HTTP_ALLOW_PIPELINING)
++        allowNewPipelines = false;
++
++    // step 3: consider an idle persistent connection
++    if (allowNewPipelines && (caps & NS_HTTP_ALLOW_KEEPALIVE)) {
+         nsRefPtr<nsHttpConnection> conn;
+         while (!conn && (ent->mIdleConns.Length() > 0)) {
+             conn = ent->mIdleConns[0];
+@@ -1483,21 +1564,8 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent,
+         }
+     }
+ 
+-    // step 3
+-    // consider pipelining scripts and revalidations
+-    if (!attemptedOptimisticPipeline &&
+-        (classification == nsHttpTransaction::CLASS_REVALIDATION ||
+-         classification == nsHttpTransaction::CLASS_SCRIPT)) {
+-        attemptedOptimisticPipeline = true;
+-        if (AddToShortestPipeline(ent, trans,
+-                                  classification,
+-                                  mMaxOptimisticPipelinedRequests)) {
+-            return NS_OK;
+-        }
+-    }
+-
+-    // step 4
+-    if (!onlyReusedConnection) {
++    // step 4: Maybe make a connection? 
++    if (!onlyReusedConnection && allowNewPipelines) {
+         nsresult rv = MakeNewConnection(ent, trans);
+         if (NS_SUCCEEDED(rv)) {
+             // this function returns NOT_AVAILABLE for asynchronous connects
+@@ -1510,17 +1578,16 @@ nsHttpConnectionMgr::TryDispatchTransaction(nsConnectionEntry *ent,
+             return rv;
+         }
+     }
++
++    // XXX: We dequeue and queue the same url here sometimes..
++#ifdef WTF_DEBUG
++    nsHttpRequestHead *head = trans->RequestHead();
++    fprintf(stderr, "WTF: Queuing url %s%s\n",
++            ent->mConnInfo->Host(),
++            head ? head->RequestURI().BeginReading() : "<unknown?>");
++#endif
+     
+-    // step 5
+-    if (caps & NS_HTTP_ALLOW_PIPELINING) {
+-        if (AddToShortestPipeline(ent, trans,
+-                                  classification,
+-                                  mMaxPipelinedRequests)) {
+-            return NS_OK;
+-        }
+-    }
+-    
+-    // step 6
++    // step 6: Queue it
+     return NS_ERROR_NOT_AVAILABLE;                /* queue it */
+ }
+ 
+@@ -1590,10 +1657,20 @@ nsHttpConnectionMgr::DispatchAbstractTransaction(nsConnectionEntry *ent,
+         if (!NS_SUCCEEDED(rv))
+             return rv;
+         transaction = pipeline;
++#ifdef WTF_DEBUG
++        fprintf(stderr, "WTF: New pipeline created from %d idle conns for host %s\n",
++                ent->mIdleConns.Length(), ent->mConnInfo->Host());
++#endif
+     }
+     else {
+         LOG(("   not using pipeline datastructure due to class solo.\n"));
+         transaction = aTrans;
++#ifdef WTF_TEST
++        nsHttpRequestHead *head = transaction->RequestHead();
++        fprintf(stderr, "WTF-order: Pipeline forbidden for url %s%s\n",
++                ent->mConnInfo->Host(),
++                head ? head->RequestURI().BeginReading() : "<unknown?>");
++#endif
+     }
+ 
+     nsRefPtr<nsConnectionHandle> handle = new nsConnectionHandle(conn);
+@@ -1692,27 +1769,15 @@ nsHttpConnectionMgr::ProcessNewTransaction(nsHttpTransaction *trans)
+                           "Sticky Connection Not In Active List");
+         trans->SetConnection(nullptr);
+         rv = DispatchTransaction(ent, trans, conn);
+-    }
+-    else
+-        rv = TryDispatchTransaction(ent, false, trans);
+-
+-    if (NS_SUCCEEDED(rv)) {
+-        LOG(("  ProcessNewTransaction Dispatch Immediately trans=%p\n", trans));
+         return rv;
+     }
+-    
+-    if (rv == NS_ERROR_NOT_AVAILABLE) {
+-        LOG(("  adding transaction to pending queue "
+-             "[trans=%p pending-count=%u]\n",
+-             trans, ent->mPendingQ.Length()+1));
+-        // put this transaction on the pending queue...
++    else {
++        // XXX: maybe check the queue first and directly call TryDispatch?
+         InsertTransactionSorted(ent->mPendingQ, trans);
+         NS_ADDREF(trans);
++        ProcessPendingQForEntry(ent);
+         return NS_OK;
+     }
+-
+-    LOG(("  ProcessNewTransaction Hard Error trans=%p rv=%x\n", trans, rv));
+-    return rv;
+ }
+ 
+ 
+@@ -2311,13 +2376,37 @@ nsHttpConnectionMgr::OnMsgSpeculativeConnect(int32_t, void *param)
+     if (preferredEntry)
+         ent = preferredEntry;
  
++    /* Only speculative connect if we're not pipelining */
+     if (!ent->mIdleConns.Length() && !RestrictConnections(ent) &&
+-        !AtActiveConnectionLimit(ent, trans->Caps())) {
++        !HasPipelines(ent) && !AtActiveConnectionLimit(ent, trans->Caps())) {
+         CreateTransport(ent, trans, trans->Caps(), true);
+     }
+ }
+ 
+ bool
++nsHttpConnectionMgr::HasPipelines(nsConnectionEntry *ent)
++{
++    uint32_t activeCount = ent->mActiveConns.Length();
++
++    if (!ent->SupportsPipelining()) {
++        return false;
++    }
++
++    for (uint32_t i = 0; i < activeCount; ++i) {
++        nsHttpConnection *conn = ent->mActiveConns[i];
++        if (!conn->SupportsPipelining())
++            continue;
++
++        nsAHttpTransaction *activeTrans = conn->Transaction();
++
++        if (activeTrans && !activeTrans->IsDone() &&
++            !NS_FAILED(activeTrans->Status()))
++            return true;
++    }
++    return false;
++}
++
++bool
+ nsHttpConnectionMgr::nsConnectionHandle::IsPersistent()
+ {
+     return mConn->IsPersistent();
+@@ -2852,9 +2941,12 @@ nsConnectionEntry::nsConnectionEntry(nsHttpConnectionInfo *ci)
+ {
+     NS_ADDREF(mConnInfo);
+     if (gHttpHandler->GetPipelineAggressive()) {
+-        mGreenDepth = kPipelineUnlimited;
++        // Randomize the pipeline depth (3..32)
++        mGreenDepth = gHttpHandler->GetMaxOptimisticPipelinedRequests()
++                      + rand() % gHttpHandler->GetMaxPipelinedRequests();
+         mPipelineState = PS_GREEN;
+     }
++
+     mInitialGreenDepth = mGreenDepth;
+     memset(mPipeliningClassPenalty, 0, sizeof(int16_t) * nsAHttpTransaction::CLASS_MAX);
+ }
+@@ -2892,8 +2984,9 @@ nsConnectionEntry::OnPipelineFeedbackInfo(
+         LOG(("Transaction completed at pipeline depth of %d. Host = %s\n",
+              depth, mConnInfo->Host()));
+ 
+-        if (depth >= 3)
+-            mGreenDepth = kPipelineUnlimited;
++        // Don't set this. We want to keep our initial random value..
++        //if (depth >= 3)
++        //    mGreenDepth = kPipelineUnlimited;
+     }
+ 
+     nsAHttpTransaction::Classifier classification;
+@@ -2921,6 +3014,11 @@ nsConnectionEntry::OnPipelineFeedbackInfo(
+                  mPipelineState, mConnInfo->Host()));
+             mPipelineState = PS_RED;
+             mPipeliningPenalty = 0;
++#ifdef WTF_TEST
++            fprintf(stderr, "WTF-bad: Red pipeline status disabled host %s\n",
++                    mConnInfo->Host());
++#endif
++
+         }
+ 
+         if (mLastCreditTime.IsNull())
 diff --git a/netwerk/protocol/http/nsHttpConnectionMgr.h b/netwerk/protocol/http/nsHttpConnectionMgr.h
-index 580710a..b22c669 100644
+index 580710a..7aecb68 100644
 --- a/netwerk/protocol/http/nsHttpConnectionMgr.h
 +++ b/netwerk/protocol/http/nsHttpConnectionMgr.h
-@@ -23,6 +23,7 @@
+@@ -23,11 +23,23 @@
  #include "nsIObserver.h"
  #include "nsITimer.h"
  #include "nsIX509Cert3.h"
@@ -145,15 +505,187 @@ index 580710a..b22c669 100644
  
  class nsHttpPipeline;
  
-@@ -585,6 +586,8 @@ private:
-     uint64_t mTimeOfNextWakeUp;
-     // Timer for next pruning of dead connections.
-     nsCOMPtr<nsITimer> mTimer;
-+    // Random number generator for reordering HTTP pipeline
-+    nsCOMPtr<nsIRandomGenerator>             mRandomGenerator;
+ class nsIHttpUpgradeListener;
+ 
++// We need our own optional debug define because pipelining behavior
++// is significantly altered by rendering speed (which is abysmal on
++// debug builds)
++#ifdef DEBUG
++# define WTF_DEBUG
++#endif
++
++#ifdef WTF_DEBUG
++# define WTF_TEST
++#endif
++
+ //-----------------------------------------------------------------------------
+ 
+ class nsHttpConnectionMgr : public nsIObserver
+@@ -478,6 +490,7 @@ private:
+     nsresult BuildPipeline(nsConnectionEntry *,
+                            nsAHttpTransaction *,
+                            nsHttpPipeline **);
++    bool     HasPipelines(nsConnectionEntry *);
+     bool     RestrictConnections(nsConnectionEntry *);
+     nsresult ProcessNewTransaction(nsHttpTransaction *);
+     nsresult EnsureSocketThreadTargetIfOnline();
+@@ -492,7 +505,7 @@ private:
+ 
+     nsresult MakeNewConnection(nsConnectionEntry *ent,
+                                nsHttpTransaction *trans);
+-    bool     AddToShortestPipeline(nsConnectionEntry *ent,
++    bool     AddToBestPipeline(nsConnectionEntry *ent,
+                                    nsHttpTransaction *trans,
+                                    nsHttpTransaction::Classifier classification,
+                                    uint16_t depthLimit);
+diff --git a/netwerk/protocol/http/nsHttpHandler.h b/netwerk/protocol/http/nsHttpHandler.h
+index 2963195..cd79069 100644
+--- a/netwerk/protocol/http/nsHttpHandler.h
++++ b/netwerk/protocol/http/nsHttpHandler.h
+@@ -215,6 +215,8 @@ public:
+                                      nsCString& hostLine);
+ 
+     bool GetPipelineAggressive()     { return mPipelineAggressive; }
++    uint32_t GetMaxPipelinedRequests()     { return mMaxPipelinedRequests; }
++    uint32_t GetMaxOptimisticPipelinedRequests()     { return mMaxOptimisticPipelinedRequests; }
+     void GetMaxPipelineObjectSize(int64_t *outVal)
+     {
+         *outVal = mMaxPipelineObjectSize;
+diff --git a/netwerk/protocol/http/nsHttpPipeline.cpp b/netwerk/protocol/http/nsHttpPipeline.cpp
+index 9e59878..a9e9911 100644
+--- a/netwerk/protocol/http/nsHttpPipeline.cpp
++++ b/netwerk/protocol/http/nsHttpPipeline.cpp
+@@ -87,6 +87,32 @@ nsHttpPipeline::~nsHttpPipeline()
+         free(mPushBackBuf);
+ }
+ 
++// Generate a shuffled request ordering sequence 
++void
++nsHttpPipeline::ShuffleTransOrder(uint32_t count)
++{
++   if (count < 2)
++       return;
++
++   uint32_t pos = mRequestQ[0]->PipelinePosition();
++   uint32_t i = 0;
++
++   for (i=0; i < count; ++i) {
++       uint32_t ridx = rand() % count;
++
++       nsAHttpTransaction *tmp = mRequestQ[i];
++       mRequestQ[i] = mRequestQ[ridx];
++       mRequestQ[ridx] = tmp;
++   }
++
++   for (i=0; i < count; ++i) {
++       mRequestQ[i]->SetPipelinePosition(pos);
++       pos++;
++   }
++
++   LOG(("nsHttpPipeline::ShuffleTransOrder: Shuffled %d transactions.\n", count));
++}
++
+ nsresult
+ nsHttpPipeline::AddTransaction(nsAHttpTransaction *trans)
+ {
+@@ -112,6 +138,8 @@ nsHttpPipeline::AddTransaction(nsAHttpTransaction *trans)
+     // the pipeline object.
+     trans->SetConnection(this);
+ 
++    ShuffleTransOrder(mRequestQ.Length());
++
+     if (mConnection && !mClosed && mRequestQ.Length() == 1)
+         mConnection->ResumeSend();
+ 
+@@ -760,8 +788,11 @@ nsHttpPipeline::CancelPipeline(nsresult originalReason)
+     if (respLen > 1)
+         mResponseQ.TruncateLength(1);
+ 
+-    DontReuse();
+-    Classify(nsAHttpTransaction::CLASS_SOLO);
++    /* Don't flag timed out connections as unreusable.. Tor is just slow :( */
++    if (originalReason != NS_ERROR_NET_TIMEOUT) {
++        DontReuse();
++        Classify(nsAHttpTransaction::CLASS_SOLO);
++    }
+ 
+     return total;
+ }
+@@ -842,8 +873,19 @@ nsHttpPipeline::FillSendBuf()
+ 
+     uint32_t n;
+     uint64_t avail;
++    uint64_t totalAvailable = Available();
++    uint64_t totalSent = 0;
++    uint64_t reqsSent = 0;
++    uint64_t alreadyPending = 0;
++
++    mSendBufIn->Available(&alreadyPending);
++
+     nsAHttpTransaction *trans;
+     nsITransport *transport = Transport();
++#ifdef WTF_TEST
++    nsRefPtr<nsHttpConnectionInfo> ci;
++    GetConnectionInfo(getter_AddRefs(ci));
++#endif
+ 
+     while ((trans = Request(0)) != nullptr) {
+         avail = trans->Available();
+@@ -864,6 +906,7 @@ nsHttpPipeline::FillSendBuf()
+             }
+ 
+             mSendingToProgress += n;
++            totalSent += n;
+             if (!mSuppressSendEvents && transport) {
+                 // Simulate a SENDING_TO event
+                 trans->OnTransportStatus(transport,
+@@ -874,6 +917,14 @@ nsHttpPipeline::FillSendBuf()
+ 
+         avail = trans->Available();
+         if (avail == 0) {
++#ifdef WTF_TEST
++            nsHttpRequestHead *head = trans->RequestHead();
++            fprintf(stderr, "WTF-order: Pipelined req %d/%d (%dB). Url: %s%s\n",
++                    trans->PipelinePosition(), PipelineDepth(), n,
++                    ci->Host(), head ? head->RequestURI().BeginReading() : "<unknown?>");
++#endif
++            reqsSent++;
++
+             // move transaction from request queue to response queue
+             mRequestQ.RemoveElementAt(0);
+             mResponseQ.AppendElement(trans);
+@@ -893,5 +944,12 @@ nsHttpPipeline::FillSendBuf()
+         else
+             mRequestIsPartial = true;
+     }
++
++#ifdef WTF_TEST
++    if (totalSent)
++      fprintf(stderr, "WTF-combine: Sent %d/%d bytes of %d combined pipelined requests for host %s\n",
++              alreadyPending+totalSent, totalAvailable, reqsSent, ci->Host());
++#endif
++
+     return NS_OK;
+ }
+diff --git a/netwerk/protocol/http/nsHttpPipeline.h b/netwerk/protocol/http/nsHttpPipeline.h
+index 746a196..4dc06c1 100644
+--- a/netwerk/protocol/http/nsHttpPipeline.h
++++ b/netwerk/protocol/http/nsHttpPipeline.h
+@@ -27,11 +27,14 @@ public:
+     nsHttpPipeline();
+     virtual ~nsHttpPipeline();
+ 
++    uint32_t RequestDepth() { return mRequestQ.Length(); }
++
+ private:
+     nsresult FillSendBuf();
+     
+     static NS_METHOD ReadFromPipe(nsIInputStream *, void *, const char *,
+                                   uint32_t, uint32_t, uint32_t *);
++    void ShuffleTransOrder(uint32_t);
  
-     // A 1s tick to call nsHttpConnection::ReadTimeoutTick on
-     // active http/1 connections and check for orphaned half opens.
+     // convenience functions
+     nsAHttpTransaction *Request(int32_t i)
 -- 
-1.7.5.4
+1.7.9.5
 



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits