[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [onionoo/release] Don't rely on system time for compressing histories.

To: tor-commits@xxxxxxxxxxxxxxxxxxxx
Subject: [tor-commits] [onionoo/release] Don't rely on system time for compressing histories.
From: karsten@xxxxxxxxxxxxxx
Date: Wed, 7 Feb 2018 11:20:45 +0000 (UTC)
Delivered-to: archiver@xxxxxxxx
Delivery-date: Wed, 07 Feb 2018 06:57:09 -0500
List-archive: <http://lists.torproject.org/pipermail/tor-commits/>
List-help: <mailto:tor-commits-request@lists.torproject.org?subject=help>
List-id: "auto: code repository commits" <tor-commits.lists.torproject.org>
List-post: <mailto:tor-commits@lists.torproject.org>
List-subscribe: <https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=subscribe>
List-unsubscribe: <https://lists.torproject.org/cgi-bin/mailman/options/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=unsubscribe>
Patch-author: Karsten Loesing <karsten.loesing@xxxxxxx>
Sender: "tor-commits" <tor-commits-bounces@xxxxxxxxxxxxxxxxxxxx>
commit 8b14cb159b7ee5b4f4e2c11a9a4b03269bda6974
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu Jan 11 12:45:09 2018 +0100

    Don't rely on system time for compressing histories.
    
    Related to not relying on system time for writing histories, we also
    shouldn't rely on it for compressing histories anymore. Otherwise we
    might be compressing the history of a relay or bridge that recently
    went offline while expecting an uncompressed history of that relay for
    writing history documents. In short, let's do the same when updating
    *Status objects as we do when writing *Document objects.
    
    Related to #16513, but not strictly part of it.
---
 .../torproject/onionoo/docs/BandwidthStatus.java   | 23 ++++++++++------------
 .../org/torproject/onionoo/docs/ClientsStatus.java |  7 +++----
 .../org/torproject/onionoo/docs/WeightsStatus.java | 11 +++++------
 .../onionoo/updater/BandwidthStatusUpdater.java    |  7 ++++++-
 .../onionoo/updater/ClientsStatusUpdater.java      |  7 ++++++-
 .../onionoo/updater/WeightsStatusUpdater.java      |  7 ++++++-
 .../torproject/onionoo/docs/WeightsStatusTest.java | 14 ++++++-------
 7 files changed, 43 insertions(+), 33 deletions(-)

diff --git a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java
index 1a81d20..ba22dd4 100644
--- a/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/BandwidthStatus.java
@@ -108,16 +108,13 @@ public class BandwidthStatus extends Document {
     }
   }
 
-  public void compressHistory() {
-    this.compressHistory(System.currentTimeMillis());
+  public void compressHistory(long lastSeenMillis) {
+    this.compressHistory(this.writeHistory, lastSeenMillis);
+    this.compressHistory(this.readHistory, lastSeenMillis);
   }
 
-  public void compressHistory(long now) {
-    this.compressHistory(this.writeHistory, now);
-    this.compressHistory(this.readHistory, now);
-  }
-
-  private void compressHistory(SortedMap<Long, long[]> history, long now) {
+  private void compressHistory(SortedMap<Long, long[]> history,
+      long lastSeenMillis) {
     SortedMap<Long, long[]> uncompressedHistory = new TreeMap<>(history);
     history.clear();
     long lastStartMillis = 0L;
@@ -129,17 +126,17 @@ public class BandwidthStatus extends Document {
       long endMillis = v[1];
       long bandwidth = v[2];
       long intervalLengthMillis;
-      if (now - endMillis <= DateTimeHelper.THREE_DAYS) {
+      if (lastSeenMillis - endMillis <= DateTimeHelper.THREE_DAYS) {
         intervalLengthMillis = DateTimeHelper.FIFTEEN_MINUTES;
-      } else if (now - endMillis <= DateTimeHelper.ONE_WEEK) {
+      } else if (lastSeenMillis - endMillis <= DateTimeHelper.ONE_WEEK) {
         intervalLengthMillis = DateTimeHelper.ONE_HOUR;
-      } else if (now - endMillis
+      } else if (lastSeenMillis - endMillis
           <= DateTimeHelper.ROUGHLY_ONE_MONTH) {
         intervalLengthMillis = DateTimeHelper.FOUR_HOURS;
-      } else if (now - endMillis
+      } else if (lastSeenMillis - endMillis
           <= DateTimeHelper.ROUGHLY_THREE_MONTHS) {
         intervalLengthMillis = DateTimeHelper.TWELVE_HOURS;
-      } else if (now - endMillis
+      } else if (lastSeenMillis - endMillis
           <= DateTimeHelper.ROUGHLY_ONE_YEAR) {
         intervalLengthMillis = DateTimeHelper.TWO_DAYS;
       } else {
diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java
index 5310abf..a5bc8e8 100644
--- a/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/ClientsStatus.java
@@ -69,18 +69,17 @@ public class ClientsStatus extends Document {
 
   /** Compresses the history of clients objects by merging adjacent
    * intervals, depending on how far back in the past they lie. */
-  public void compressHistory() {
+  public void compressHistory(long lastSeenMillis) {
     SortedSet<ClientsHistory> uncompressedHistory = new TreeSet<>(this.history);
     history.clear();
     ClientsHistory lastResponses = null;
     String lastMonthString = "1970-01";
-    long now = System.currentTimeMillis();
     for (ClientsHistory responses : uncompressedHistory) {
       long intervalLengthMillis;
-      if (now - responses.getEndMillis()
+      if (lastSeenMillis - responses.getEndMillis()
           <= DateTimeHelper.ROUGHLY_THREE_MONTHS) {
         intervalLengthMillis = DateTimeHelper.ONE_DAY;
-      } else if (now - responses.getEndMillis()
+      } else if (lastSeenMillis - responses.getEndMillis()
           <= DateTimeHelper.ROUGHLY_ONE_YEAR) {
         intervalLengthMillis = DateTimeHelper.TWO_DAYS;
       } else {
diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java
index 0dfd4f1..94efda6 100644
--- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java
@@ -125,7 +125,7 @@ public class WeightsStatus extends Document {
 
   /** Compresses the history of weights objects by merging adjacent
    * intervals, depending on how far back in the past they lie. */
-  public void compressHistory() {
+  public void compressHistory(long lastSeenMillis) {
     SortedMap<long[], double[]> uncompressedHistory =
         new TreeMap<>(histComparator);
     uncompressedHistory.putAll(this.history);
@@ -135,21 +135,20 @@ public class WeightsStatus extends Document {
     double[] lastWeights = null;
     String lastMonthString = "1970-01";
     int lastMissingValues = -1;
-    long now = System.currentTimeMillis();
     for (Map.Entry<long[], double[]> e : uncompressedHistory.entrySet()) {
       long startMillis = e.getKey()[0];
       long endMillis = e.getKey()[1];
       double[] weights = e.getValue();
       long intervalLengthMillis;
-      if (now - endMillis <= DateTimeHelper.ONE_WEEK) {
+      if (lastSeenMillis - endMillis <= DateTimeHelper.ONE_WEEK) {
         intervalLengthMillis = DateTimeHelper.ONE_HOUR;
-      } else if (now - endMillis
+      } else if (lastSeenMillis - endMillis
           <= DateTimeHelper.ROUGHLY_ONE_MONTH) {
         intervalLengthMillis = DateTimeHelper.FOUR_HOURS;
-      } else if (now - endMillis
+      } else if (lastSeenMillis - endMillis
           <= DateTimeHelper.ROUGHLY_THREE_MONTHS) {
         intervalLengthMillis = DateTimeHelper.TWELVE_HOURS;
-      } else if (now - endMillis
+      } else if (lastSeenMillis - endMillis
           <= DateTimeHelper.ROUGHLY_ONE_YEAR) {
         intervalLengthMillis = DateTimeHelper.TWO_DAYS;
       } else {
diff --git a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java
index d8288f8..b5e5ddb 100644
--- a/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/BandwidthStatusUpdater.java
@@ -8,6 +8,7 @@ import org.torproject.descriptor.ExtraInfoDescriptor;
 import org.torproject.onionoo.docs.BandwidthStatus;
 import org.torproject.onionoo.docs.DocumentStore;
 import org.torproject.onionoo.docs.DocumentStoreFactory;
+import org.torproject.onionoo.docs.NodeStatus;
 
 public class BandwidthStatusUpdater implements DescriptorListener,
     StatusUpdater {
@@ -58,7 +59,11 @@ public class BandwidthStatusUpdater implements DescriptorListener,
       bandwidthStatus.addToReadHistory(descriptor.getReadHistory());
     }
     if (bandwidthStatus.isDirty()) {
-      bandwidthStatus.compressHistory();
+      NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class,
+          true, fingerprint);
+      if (null != nodeStatus) {
+        bandwidthStatus.compressHistory(nodeStatus.getLastSeenMillis());
+      }
       this.documentStore.store(bandwidthStatus, fingerprint);
       bandwidthStatus.clearDirty();
     }
diff --git a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java
index 0021068..0529dcf 100644
--- a/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/ClientsStatusUpdater.java
@@ -10,6 +10,7 @@ import org.torproject.onionoo.docs.ClientsStatus;
 import org.torproject.onionoo.docs.DateTimeHelper;
 import org.torproject.onionoo.docs.DocumentStore;
 import org.torproject.onionoo.docs.DocumentStoreFactory;
+import org.torproject.onionoo.docs.NodeStatus;
 import org.torproject.onionoo.util.FormattingUtils;
 
 import java.util.Map;
@@ -158,7 +159,11 @@ public class ClientsStatusUpdater implements DescriptorListener,
       }
       clientsStatus.addToHistory(e.getValue());
       if (clientsStatus.isDirty()) {
-        clientsStatus.compressHistory();
+        NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class,
+            true, hashedFingerprint);
+        if (null != nodeStatus) {
+          clientsStatus.compressHistory(nodeStatus.getLastSeenMillis());
+        }
         this.documentStore.store(clientsStatus, hashedFingerprint);
         clientsStatus.clearDirty();
       }
diff --git a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java
index 6c5ec0d..886f2f8 100644
--- a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java
@@ -8,6 +8,7 @@ import org.torproject.descriptor.NetworkStatusEntry;
 import org.torproject.descriptor.RelayNetworkStatusConsensus;
 import org.torproject.onionoo.docs.DocumentStore;
 import org.torproject.onionoo.docs.DocumentStoreFactory;
+import org.torproject.onionoo.docs.NodeStatus;
 import org.torproject.onionoo.docs.WeightsStatus;
 
 import java.util.Arrays;
@@ -76,7 +77,11 @@ public class WeightsStatusUpdater implements DescriptorListener,
       weightsStatus.addToHistory(validAfterMillis, freshUntilMillis,
           weights);
       if (weightsStatus.isDirty()) {
-        weightsStatus.compressHistory();
+        NodeStatus nodeStatus = this.documentStore.retrieve(NodeStatus.class,
+            true, fingerprint);
+        if (null != nodeStatus) {
+          weightsStatus.compressHistory(nodeStatus.getLastSeenMillis());
+        }
         this.documentStore.store(weightsStatus, fingerprint);
         weightsStatus.clearDirty();
       }
diff --git a/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java b/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java
index c0457e7..e28f989 100644
--- a/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java
+++ b/src/test/java/org/torproject/onionoo/docs/WeightsStatusTest.java
@@ -38,16 +38,16 @@ public class WeightsStatusTest {
     WeightsStatus ws = new WeightsStatus();
     SortedMap<long[], double[]> history = ws.getHistory();
     assertTrue("actually: " + mapToString(history), history.isEmpty());
-    long now = System.currentTimeMillis();
+    long lastSeenMillis = 1515670935476L;
     long hourMillis = 60L * 60L * 1000L;
     for (long j = 1L; j < 10L; j++) { // add arbitrary data
-      ws.addToHistory(now - j * hourMillis - hourMillis,
-          now - j * hourMillis,
+      ws.addToHistory(lastSeenMillis - j * hourMillis - hourMillis,
+          lastSeenMillis - j * hourMillis,
           new double[]{ 1.0 * j, 2.0 * j, 1.0 * j, 2.0 * j, 1.0 * j,
               2.0 * j, Double.NaN});
       assertEquals("have: " + mapToString(history), (int) j, history.size());
     }
-    ws.compressHistory();
+    ws.compressHistory(lastSeenMillis);
     assertEquals("history map: " + mapToString(history), 9, history.size());
     assertFalse("document shouldn't contain NaN: " + ws.toDocumentString(),
         ws.toDocumentString().contains("NaN"));
@@ -67,7 +67,7 @@ public class WeightsStatusTest {
               2.0 * j, j * 0.5});
       assertEquals("have: " + mapToString(history), (int) j, history.size());
     }
-    ws.compressHistory();
+    ws.compressHistory(1515670935476L);
     assertEquals("history map: " + mapToString(history), 1, history.size());
     assertFalse("document shouldn't contain NaN: " + ws.toDocumentString(),
         ws.toDocumentString().contains("NaN"));
@@ -101,7 +101,7 @@ public class WeightsStatusTest {
     }
     assertEquals("history: ", correctHistory, mapToString(ws.getHistory()));
     assertEquals(correctLines.length, ws.getHistory().size());
-    ws.compressHistory();
+    ws.compressHistory(1515670935476L);
     assertEquals("found: " + mapToString(ws.getHistory()), 1,
         ws.getHistory().size());
     assertEquals("[1431032400000, 1431043200000] : [-1.0, 1.78279826E-4, "
@@ -128,7 +128,7 @@ public class WeightsStatusTest {
       assertEquals("document string: ", exp, ws.toDocumentString());
     }
     assertEquals(correctLines.length, ws.getHistory().size());
-    ws.compressHistory();
+    ws.compressHistory(1515670935476L);
     assertEquals("found: " + mapToString(ws.getHistory()), 3,
         ws.getHistory().size());
   }



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
Next by Author: [tor-commits] [metrics-lib/master] Store size as simple int.
Previous by thread: [tor-commits] [onionoo/release] Rename lastSeenMillis to mostRecentStatusMillis.
Next by thread: [tor-commits] [onionoo/release] Bump version to 5.0-1.9.0-dev.
Index(es):
- Author
- Thread