[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [onionoo/master] Add fractional time of having a flag assigned.



commit 43434dc5f4b8f9c588accc6c4e8e0b8981becc98
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu Mar 19 22:44:20 2015 +0100

    Add fractional time of having a flag assigned.
    
    Implements #15177.
---
 build.xml                                          |    4 +-
 .../torproject/onionoo/docs/UptimeDocument.java    |   10 ++
 .../org/torproject/onionoo/docs/UptimeHistory.java |   37 ++++--
 .../org/torproject/onionoo/docs/UptimeStatus.java  |   74 +++++++-----
 .../torproject/onionoo/server/ResponseBuilder.java |    2 +-
 .../onionoo/updater/UptimeStatusUpdater.java       |   94 +++++++++++----
 .../onionoo/writer/UptimeDocumentWriter.java       |   42 ++++++-
 .../torproject/onionoo/docs/UptimeStatusTest.java  |  123 +++++++++++++++-----
 .../torproject/onionoo/updater/DummyConsensus.java |    7 +-
 .../onionoo/updater/UptimeStatusUpdaterTest.java   |    5 +-
 web/protocol.html                                  |   19 +++
 11 files changed, 319 insertions(+), 98 deletions(-)

diff --git a/build.xml b/build.xml
index a3f105e..5986416 100644
--- a/build.xml
+++ b/build.xml
@@ -1,8 +1,8 @@
 <project default="dist" name="onionoo" basedir=".">
 
-  <property name="onionoo.protocol.version" value="2.2"/>
+  <property name="onionoo.protocol.version" value="2.3"/>
   <property name="release.version"
-            value="${onionoo.protocol.version}.1"/>
+            value="${onionoo.protocol.version}.0"/>
   <property name="javasources" value="src/main/java"/>
   <property name="tests" value="src/test/java"/>
   <property name="classes" value="classes"/>
diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java b/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java
index 7f0bacc..9b84a31 100644
--- a/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java
+++ b/src/main/java/org/torproject/onionoo/docs/UptimeDocument.java
@@ -3,6 +3,7 @@
 package org.torproject.onionoo.docs;
 
 import java.util.Map;
+import java.util.SortedMap;
 
 public class UptimeDocument extends Document {
 
@@ -19,5 +20,14 @@ public class UptimeDocument extends Document {
   public Map<String, GraphHistory> getUptime() {
     return this.uptime;
   }
+
+  private SortedMap<String, Map<String, GraphHistory>> flags;
+  public void setFlags(
+      SortedMap<String, Map<String, GraphHistory>> flags) {
+    this.flags = flags;
+  }
+  public SortedMap<String, Map<String, GraphHistory>> getFlags() {
+    return this.flags;
+  }
 }
 
diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java
index b686c7e..e98f72a 100644
--- a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java
+++ b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java
@@ -1,5 +1,8 @@
 package org.torproject.onionoo.docs;
 
+import java.util.SortedSet;
+import java.util.TreeSet;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -24,27 +27,33 @@ public class UptimeHistory implements Comparable<UptimeHistory> {
     return this.uptimeHours;
   }
 
+  private SortedSet<String> flags;
+  public SortedSet<String> getFlags() {
+    return this.flags;
+  }
+
   UptimeHistory(boolean relay, long startMillis,
-      int uptimeHours) {
+      int uptimeHours, SortedSet<String> flags) {
     this.relay = relay;
     this.startMillis = startMillis;
     this.uptimeHours = uptimeHours;
+    this.flags = flags;
   }
 
   public static UptimeHistory fromString(String uptimeHistoryString) {
-    String[] parts = uptimeHistoryString.split(" ", 3);
-    if (parts.length != 3) {
+    String[] parts = uptimeHistoryString.split(" ", -1);
+    if (parts.length < 3) {
       log.warn("Invalid number of space-separated strings in uptime "
           + "history: '" + uptimeHistoryString + "'.  Skipping");
       return null;
     }
     boolean relay = false;
-    if (parts[0].equals("r")) {
+    if (parts[0].equalsIgnoreCase("r")) {
       relay = true;
     } else if (!parts[0].equals("b")) {
       log.warn("Invalid node type in uptime history: '"
-          + uptimeHistoryString + "'.  Supported types are 'r' and 'b'.  "
-          + "Skipping.");
+          + uptimeHistoryString + "'.  Supported types are 'r', 'R', and "
+          + "'b'.  Skipping.");
       return null;
     }
     long startMillis = DateTimeHelper.parse(parts[1],
@@ -62,15 +71,27 @@ public class UptimeHistory implements Comparable<UptimeHistory> {
           + uptimeHistoryString + "'.  Skipping.");
       return null;
     }
-    return new UptimeHistory(relay, startMillis, uptimeHours);
+    SortedSet<String> flags = null;
+    if (parts[0].equals("R")) {
+      flags = new TreeSet<String>();
+      for (int i = 3; i < parts.length; i++) {
+        flags.add(parts[i]);
+      }
+    }
+    return new UptimeHistory(relay, startMillis, uptimeHours, flags);
   }
 
   public String toString() {
     StringBuilder sb = new StringBuilder();
-    sb.append(this.relay ? "r" : "b");
+    sb.append(this.relay ? (this.flags == null ? "r" : "R") : "b");
     sb.append(" " + DateTimeHelper.format(this.startMillis,
         DateTimeHelper.DATEHOUR_NOSPACE_FORMAT));
     sb.append(" " + String.format("%d", this.uptimeHours));
+    if (this.flags != null) {
+      for (String flag : this.flags) {
+        sb.append(" " + flag);
+      }
+    }
     return sb.toString();
   }
 
diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java b/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java
index e48fd09..266a245 100644
--- a/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java
+++ b/src/main/java/org/torproject/onionoo/docs/UptimeStatus.java
@@ -2,6 +2,7 @@
  * See LICENSE for licensing information */
 package org.torproject.onionoo.docs;
 
+import java.util.NavigableSet;
 import java.util.Scanner;
 import java.util.SortedSet;
 import java.util.TreeSet;
@@ -24,18 +25,12 @@ public class UptimeStatus extends Document {
 
   private SortedSet<UptimeHistory> relayHistory =
       new TreeSet<UptimeHistory>();
-  public void setRelayHistory(SortedSet<UptimeHistory> relayHistory) {
-    this.relayHistory = relayHistory;
-  }
   public SortedSet<UptimeHistory> getRelayHistory() {
     return this.relayHistory;
   }
 
   private SortedSet<UptimeHistory> bridgeHistory =
       new TreeSet<UptimeHistory>();
-  public void setBridgeHistory(SortedSet<UptimeHistory> bridgeHistory) {
-    this.bridgeHistory = bridgeHistory;
-  }
   public SortedSet<UptimeHistory> getBridgeHistory() {
     return this.bridgeHistory;
   }
@@ -59,30 +54,50 @@ public class UptimeStatus extends Document {
     s.close();
   }
 
-  public void addToHistory(boolean relay, SortedSet<Long> newIntervals) {
-    for (long startMillis : newIntervals) {
-      SortedSet<UptimeHistory> history = relay ? this.relayHistory
-          : this.bridgeHistory;
-      UptimeHistory interval = new UptimeHistory(relay, startMillis, 1);
-      if (!history.headSet(interval).isEmpty()) {
-        UptimeHistory prev = history.headSet(interval).last();
-        if (prev.isRelay() == interval.isRelay() &&
-            prev.getStartMillis() + DateTimeHelper.ONE_HOUR
-            * prev.getUptimeHours() > interval.getStartMillis()) {
-          continue;
-        }
+  public void addToHistory(boolean relay, long startMillis,
+      SortedSet<String> flags) {
+    SortedSet<UptimeHistory> history = relay ? this.relayHistory
+        : this.bridgeHistory;
+    UptimeHistory interval = new UptimeHistory(relay, startMillis, 1,
+        flags);
+    NavigableSet<UptimeHistory> existingIntervals =
+        new TreeSet<UptimeHistory>(history.headSet(new UptimeHistory(
+        relay, startMillis + DateTimeHelper.ONE_HOUR, 0, flags)));
+    for (UptimeHistory prev : existingIntervals.descendingSet()) {
+      if (prev.isRelay() != interval.isRelay() ||
+          prev.getStartMillis() + DateTimeHelper.ONE_HOUR
+          * prev.getUptimeHours() <= interval.getStartMillis()) {
+        break;
       }
-      if (!history.tailSet(interval).isEmpty()) {
-        UptimeHistory next = history.tailSet(interval).first();
-        if (next.isRelay() == interval.isRelay() &&
-            next.getStartMillis() < interval.getStartMillis()
-            + DateTimeHelper.ONE_HOUR) {
-          continue;
-        }
+      if (prev.getFlags() == interval.getFlags() ||
+          (prev.getFlags() != null && interval.getFlags() != null &&
+          prev.getFlags().equals(interval.getFlags()))) {
+        /* The exact same interval is already contained in history. */
+        return;
+      }
+      /* There is an interval that includes the new interval, but it
+       * contains different flags.  Remove the old interval, put in any
+       * parts before or after the new interval, and add the new interval
+       * further down below. */
+      history.remove(prev);
+      int hoursBefore = (int) ((interval.getStartMillis()
+          - prev.getStartMillis()) / DateTimeHelper.ONE_HOUR);
+      if (hoursBefore > 0) {
+        history.add(new UptimeHistory(relay,
+            prev.getStartMillis(), hoursBefore, prev.getFlags()));
+      }
+      int hoursAfter = (int) (prev.getStartMillis()
+          / DateTimeHelper.ONE_HOUR + prev.getUptimeHours() -
+          interval.getStartMillis() / DateTimeHelper.ONE_HOUR - 1);
+      if (hoursAfter > 0) {
+        history.add(new UptimeHistory(relay,
+            interval.getStartMillis() + DateTimeHelper.ONE_HOUR,
+            hoursAfter, prev.getFlags()));
       }
-      history.add(interval);
-      this.isDirty = true;
+      break;
     }
+    history.add(interval);
+    this.isDirty = true;
   }
 
   public void compressHistory() {
@@ -99,7 +114,10 @@ public class UptimeStatus extends Document {
       if (lastInterval != null &&
           lastInterval.getStartMillis() + DateTimeHelper.ONE_HOUR
           * lastInterval.getUptimeHours() == interval.getStartMillis() &&
-          lastInterval.isRelay() == interval.isRelay()) {
+          lastInterval.isRelay() == interval.isRelay() &&
+          (lastInterval.getFlags() == interval.getFlags() ||
+          (lastInterval.getFlags() != null && interval.getFlags() != null
+          && lastInterval.getFlags().equals(interval.getFlags())))) {
         lastInterval.addUptime(interval);
       } else {
         if (lastInterval != null) {
diff --git a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java
index d1e7f23..af0f67e 100644
--- a/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java
+++ b/src/main/java/org/torproject/onionoo/server/ResponseBuilder.java
@@ -70,7 +70,7 @@ public class ResponseBuilder {
     return this.charsWritten;
   }
 
-  private static final String PROTOCOL_VERSION = "2.2";
+  private static final String PROTOCOL_VERSION = "2.3";
 
   private static final String NEXT_MAJOR_VERSION_SCHEDULED = null;
 
diff --git a/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java
index c3c98ed..e2cee78 100644
--- a/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java
+++ b/src/main/java/org/torproject/onionoo/updater/UptimeStatusUpdater.java
@@ -2,6 +2,8 @@
  * See LICENSE for licensing information */
 package org.torproject.onionoo.updater;
 
+import java.util.BitSet;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.SortedMap;
 import java.util.SortedSet;
@@ -48,41 +50,70 @@ public class UptimeStatusUpdater implements DescriptorListener,
     }
   }
 
-  private SortedSet<Long> newRelayStatuses = new TreeSet<Long>(),
-      newBridgeStatuses = new TreeSet<Long>();
+  private static class Flags {
+
+    private static Map<String, Integer> flagIndexes =
+        new HashMap<String, Integer>();
+
+    private static Map<Integer, String> flagStrings =
+        new HashMap<Integer, String>();
+
+    private BitSet flags;
+
+    private Flags(SortedSet<String> flags) {
+      this.flags = new BitSet(flagIndexes.size());
+      for (String flag : flags) {
+        if (!flagIndexes.containsKey(flag)) {
+          flagStrings.put(flagIndexes.size(), flag);
+          flagIndexes.put(flag, flagIndexes.size());
+        }
+        this.flags.set(flagIndexes.get(flag));
+      }
+    }
+
+    public SortedSet<String> getFlags() {
+      SortedSet<String> result = new TreeSet<String>();
+      if (this.flags != null) {
+        for (int i = this.flags.nextSetBit(0); i >= 0;
+            i = this.flags.nextSetBit(i + 1)) {
+          result.add(flagStrings.get(i));
+        }
+      }
+      return result;
+    }
+  }
+
+  private SortedMap<Long, Flags>
+      newRelayStatuses = new TreeMap<Long, Flags>();
+  private SortedMap<String, SortedMap<Long, Flags>>
+      newRunningRelays = new TreeMap<String, SortedMap<Long, Flags>>();
+  private SortedSet<Long> newBridgeStatuses = new TreeSet<Long>();
   private SortedMap<String, SortedSet<Long>>
-      newRunningRelays = new TreeMap<String, SortedSet<Long>>(),
       newRunningBridges = new TreeMap<String, SortedSet<Long>>();
 
   private void processRelayNetworkStatusConsensus(
       RelayNetworkStatusConsensus consensus) {
-    SortedSet<String> fingerprints = new TreeSet<String>();
+    long dateHourMillis = (consensus.getValidAfterMillis()
+        / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR;
     for (NetworkStatusEntry entry :
         consensus.getStatusEntries().values()) {
-      if (entry.getFlags().contains("Running")) {
-        fingerprints.add(entry.getFingerprint());
-      }
-    }
-    if (!fingerprints.isEmpty()) {
-      long dateHourMillis = (consensus.getValidAfterMillis()
-          / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR;
-      for (String fingerprint : fingerprints) {
-        if (!this.newRunningRelays.containsKey(fingerprint)) {
-          this.newRunningRelays.put(fingerprint, new TreeSet<Long>());
-        }
-        this.newRunningRelays.get(fingerprint).add(dateHourMillis);
+      String fingerprint = entry.getFingerprint();
+      if (!this.newRunningRelays.containsKey(fingerprint)) {
+        this.newRunningRelays.put(fingerprint,
+            new TreeMap<Long, Flags>());
       }
-      this.newRelayStatuses.add(dateHourMillis);
+      this.newRunningRelays.get(fingerprint).put(dateHourMillis,
+          new Flags(entry.getFlags()));
     }
+    this.newRelayStatuses.put(dateHourMillis,
+        new Flags(consensus.getKnownFlags()));
   }
 
   private void processBridgeNetworkStatus(BridgeNetworkStatus status) {
     SortedSet<String> fingerprints = new TreeSet<String>();
     for (NetworkStatusEntry entry :
         status.getStatusEntries().values()) {
-      if (entry.getFlags().contains("Running")) {
-        fingerprints.add(entry.getFingerprint());
-      }
+      fingerprints.add(entry.getFingerprint());
     }
     if (!fingerprints.isEmpty()) {
       long dateHourMillis = (status.getPublishedMillis()
@@ -98,20 +129,30 @@ public class UptimeStatusUpdater implements DescriptorListener,
   }
 
   public void updateStatuses() {
-    for (Map.Entry<String, SortedSet<Long>> e :
+    for (Map.Entry<String, SortedMap<Long, Flags>> e :
         this.newRunningRelays.entrySet()) {
       this.updateStatus(true, e.getKey(), e.getValue());
     }
     this.updateStatus(true, null, this.newRelayStatuses);
     for (Map.Entry<String, SortedSet<Long>> e :
         this.newRunningBridges.entrySet()) {
-      this.updateStatus(false, e.getKey(), e.getValue());
+      SortedMap<Long, Flags> dateHourMillisNoFlags =
+          new TreeMap<Long, Flags>();
+      for (long dateHourMillis : e.getValue()) {
+        dateHourMillisNoFlags.put(dateHourMillis, null);
+      }
+      this.updateStatus(false, e.getKey(), dateHourMillisNoFlags);
+    }
+    SortedMap<Long, Flags> dateHourMillisNoFlags =
+        new TreeMap<Long, Flags>();
+    for (long dateHourMillis : this.newBridgeStatuses) {
+      dateHourMillisNoFlags.put(dateHourMillis, null);
     }
-    this.updateStatus(false, null, this.newBridgeStatuses);
+    this.updateStatus(false, null, dateHourMillisNoFlags);
   }
 
   private void updateStatus(boolean relay, String fingerprint,
-      SortedSet<Long> newUptimeHours) {
+      SortedMap<Long, Flags> dateHourMillisFlags) {
     UptimeStatus uptimeStatus = (fingerprint == null) ?
         this.documentStore.retrieve(UptimeStatus.class, true) :
         this.documentStore.retrieve(UptimeStatus.class, true,
@@ -119,7 +160,10 @@ public class UptimeStatusUpdater implements DescriptorListener,
     if (uptimeStatus == null) {
       uptimeStatus = new UptimeStatus();
     }
-    uptimeStatus.addToHistory(relay, newUptimeHours);
+    for (Map.Entry<Long, Flags> e : dateHourMillisFlags.entrySet()) {
+      uptimeStatus.addToHistory(relay, e.getKey(),
+          e.getValue() == null ? null : e.getValue().getFlags());
+    }
     if (uptimeStatus.isDirty()) {
       uptimeStatus.compressHistory();
       if (fingerprint == null) {
diff --git a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
index c5f71a1..d9eb91f 100644
--- a/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
+++ b/src/main/java/org/torproject/onionoo/writer/UptimeDocumentWriter.java
@@ -6,7 +6,10 @@ import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.SortedMap;
 import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -107,18 +110,45 @@ public class UptimeDocumentWriter implements DocumentWriter {
         this.graphIntervals.length; graphIntervalIndex++) {
       String graphName = this.graphNames[graphIntervalIndex];
       GraphHistory graphHistory = this.compileUptimeHistory(
-          graphIntervalIndex, relay, history, knownStatuses);
+          graphIntervalIndex, relay, history, knownStatuses, null);
       if (graphHistory != null) {
         uptime.put(graphName, graphHistory);
       }
     }
     uptimeDocument.setUptime(uptime);
+    SortedMap<String, Map<String, GraphHistory>> flags =
+        new TreeMap<String, Map<String, GraphHistory>>();
+    SortedSet<String> allFlags = new TreeSet<String>();
+    for (UptimeHistory hist : history) {
+      if (hist.getFlags() != null) {
+        allFlags.addAll(hist.getFlags());
+      }
+    }
+    for (String flag : allFlags) {
+      Map<String, GraphHistory> graphsForFlags =
+          new LinkedHashMap<String, GraphHistory>();
+      for (int graphIntervalIndex = 0; graphIntervalIndex <
+          this.graphIntervals.length; graphIntervalIndex++) {
+        String graphName = this.graphNames[graphIntervalIndex];
+        GraphHistory graphHistory = this.compileUptimeHistory(
+            graphIntervalIndex, relay, history, knownStatuses, flag);
+        if (graphHistory != null) {
+          graphsForFlags.put(graphName, graphHistory);
+        }
+      }
+      if (!graphsForFlags.isEmpty()) {
+        flags.put(flag, graphsForFlags);
+      }
+    }
+    if (!flags.isEmpty()) {
+      uptimeDocument.setFlags(flags);
+    }
     return uptimeDocument;
   }
 
   private GraphHistory compileUptimeHistory(int graphIntervalIndex,
       boolean relay, SortedSet<UptimeHistory> history,
-      SortedSet<UptimeHistory> knownStatuses) {
+      SortedSet<UptimeHistory> knownStatuses, String flag) {
     long graphInterval = this.graphIntervals[graphIntervalIndex];
     long dataPointInterval =
         this.dataPointIntervals[graphIntervalIndex];
@@ -130,7 +160,9 @@ public class UptimeDocumentWriter implements DocumentWriter {
     int uptimeHours = 0;
     long firstStatusStartMillis = -1L;
     for (UptimeHistory hist : history) {
-      if (hist.isRelay() != relay) {
+      if (hist.isRelay() != relay ||
+          (flag != null && (hist.getFlags() == null ||
+          !hist.getFlags().contains(flag)))) {
         continue;
       }
       if (firstStatusStartMillis < 0L) {
@@ -170,7 +202,9 @@ public class UptimeDocumentWriter implements DocumentWriter {
         / dataPointInterval) * dataPointInterval;
     int statusHours = -1;
     for (UptimeHistory hist : knownStatuses) {
-      if (hist.isRelay() != relay) {
+      if (hist.isRelay() != relay ||
+          (flag != null && (hist.getFlags() == null ||
+          !hist.getFlags().contains(flag)))) {
         continue;
       }
       long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR
diff --git a/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java b/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java
index b4bcdc8..7a48c25 100644
--- a/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java
+++ b/src/test/java/org/torproject/onionoo/docs/UptimeStatusTest.java
@@ -7,12 +7,10 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.util.Arrays;
+import java.util.SortedSet;
 import java.util.TreeSet;
 
 import org.junit.Test;
-import org.torproject.onionoo.docs.DateTimeHelper;
-import org.torproject.onionoo.docs.UptimeHistory;
-import org.torproject.onionoo.docs.UptimeStatus;
 
 public class UptimeStatusTest {
 
@@ -26,8 +24,8 @@ public class UptimeStatusTest {
   @Test()
   public void testSingleHourWriteToDisk() {
     UptimeStatus uptimeStatus = new UptimeStatus();
-    uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList(
-        new Long[] { DateTimeHelper.parse("2013-12-20 00:00:00") })));
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-12-20 00:00:00"), null);
     uptimeStatus.compressHistory();
     assertTrue("Changed uptime status should say it's dirty.",
         uptimeStatus.isDirty());
@@ -47,9 +45,10 @@ public class UptimeStatusTest {
   @Test()
   public void testTwoConsecutiveHours() {
     UptimeStatus uptimeStatus = new UptimeStatus();
-    uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList(
-        new Long[] { DateTimeHelper.parse("2013-12-20 00:00:00"),
-        DateTimeHelper.parse("2013-12-20 01:00:00") })));
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-12-20 00:00:00"), null);
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-12-20 01:00:00"), null);
     uptimeStatus.compressHistory();
     assertEquals("History must contain single entry.", 1,
         uptimeStatus.getRelayHistory().size());
@@ -73,9 +72,10 @@ public class UptimeStatusTest {
   public void testGabelmooFillInGaps() {
     UptimeStatus uptimeStatus = new UptimeStatus();
     uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE);
-    uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList(
-        new Long[] { DateTimeHelper.parse("2013-09-09 02:00:00"),
-        DateTimeHelper.parse("2013-12-20 00:00:00") })));
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-09-09 02:00:00"), null);
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-12-20 00:00:00"), null);
     assertEquals("Uncompressed history must contain five entries.", 5,
         uptimeStatus.getRelayHistory().size());
     uptimeStatus.compressHistory();
@@ -96,8 +96,8 @@ public class UptimeStatusTest {
   public void testAddExistingHourToIntervalStart() {
     UptimeStatus uptimeStatus = new UptimeStatus();
     uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE);
-    uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList(
-        new Long[] { DateTimeHelper.parse("2013-07-22 17:00:00") })));
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-07-22 17:00:00"), null);
     uptimeStatus.compressHistory();
     assertFalse("Unchanged history should not make uptime status dirty.",
         uptimeStatus.isDirty());
@@ -107,8 +107,8 @@ public class UptimeStatusTest {
   public void testAddExistingHourToIntervalEnd() {
     UptimeStatus uptimeStatus = new UptimeStatus();
     uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE);
-    uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList(
-        new Long[] { DateTimeHelper.parse("2013-09-09 01:00:00") })));
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-09-09 01:00:00"), null);
     uptimeStatus.compressHistory();
     assertFalse("Unchanged history should not make uptime status dirty.",
         uptimeStatus.isDirty());
@@ -118,9 +118,10 @@ public class UptimeStatusTest {
   public void testTwoHoursOverlappingWithIntervalStart() {
     UptimeStatus uptimeStatus = new UptimeStatus();
     uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE);
-    uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList(
-        new Long[] { DateTimeHelper.parse("2013-07-22 16:00:00"),
-        DateTimeHelper.parse("2013-07-22 17:00:00")})));
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-07-22 16:00:00"), null);
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-07-22 17:00:00"), null);
     uptimeStatus.compressHistory();
     assertEquals("Compressed history must still contain three entries.",
         3, uptimeStatus.getRelayHistory().size());
@@ -139,9 +140,10 @@ public class UptimeStatusTest {
   public void testTwoHoursOverlappingWithIntervalEnd() {
     UptimeStatus uptimeStatus = new UptimeStatus();
     uptimeStatus.setFromDocumentString(RELAY_UPTIME_SAMPLE);
-    uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList(
-        new Long[] { DateTimeHelper.parse("2013-09-09 01:00:00"),
-        DateTimeHelper.parse("2013-09-09 02:00:00")})));
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-09-09 01:00:00"), null);
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-09-09 02:00:00"), null);
     uptimeStatus.compressHistory();
     assertEquals("Compressed history must now contain two entries.",
         2, uptimeStatus.getRelayHistory().size());
@@ -164,9 +166,10 @@ public class UptimeStatusTest {
   public void testAddRelayUptimeHours() {
     UptimeStatus uptimeStatus = new UptimeStatus();
     uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE);
-    uptimeStatus.addToHistory(true, new TreeSet<Long>(Arrays.asList(
-        new Long[] { DateTimeHelper.parse("2013-07-22 16:00:00"),
-        DateTimeHelper.parse("2014-03-21 20:00:00")})));
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-07-22 16:00:00"), null);
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2014-03-21 20:00:00"), null);
     uptimeStatus.compressHistory();
     assertEquals("Compressed relay history must still contain one entry.",
         1, uptimeStatus.getRelayHistory().size());
@@ -185,9 +188,10 @@ public class UptimeStatusTest {
   public void testAddBridgeUptimeHours() {
     UptimeStatus uptimeStatus = new UptimeStatus();
     uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE);
-    uptimeStatus.addToHistory(false, new TreeSet<Long>(Arrays.asList(
-        new Long[] { DateTimeHelper.parse("2013-07-22 16:00:00"),
-        DateTimeHelper.parse("2014-03-21 20:00:00")})));
+    uptimeStatus.addToHistory(false,
+        DateTimeHelper.parse("2013-07-22 16:00:00"), null);
+    uptimeStatus.addToHistory(false,
+        DateTimeHelper.parse("2014-03-21 20:00:00"), null);
     uptimeStatus.compressHistory();
     assertEquals("Compressed bridge history must still contain one "
         + "entry.", 1, uptimeStatus.getBridgeHistory().size());
@@ -201,5 +205,70 @@ public class UptimeStatusTest {
     assertEquals("History uptime hours not 1+5811+1=5813.", 5813,
         newUptimeHistory.getUptimeHours());
   }
+
+  private static final SortedSet<String> RUNNING_FLAG =
+      new TreeSet<String>(Arrays.asList(new String[] { "Running" }));
+
+  @Test()
+  public void testAddFlagsToNoFlagsEnd() {
+    UptimeStatus uptimeStatus = new UptimeStatus();
+    uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE);
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2014-03-21 20:00:00"), RUNNING_FLAG);
+    uptimeStatus.compressHistory();
+    assertEquals("Mixed relay history must not be compressed.", 2,
+        uptimeStatus.getRelayHistory().size());
+  }
+
+  @Test()
+  public void testAddFlagsToNoFlagsBegin() {
+    UptimeStatus uptimeStatus = new UptimeStatus();
+    uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE);
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-07-22 16:00:00"), RUNNING_FLAG);
+    uptimeStatus.compressHistory();
+    assertEquals("Mixed relay history must not be compressed.", 2,
+        uptimeStatus.getRelayHistory().size());
+  }
+
+  @Test()
+  public void testAddFlagsToNoFlagsMiddle() {
+    UptimeStatus uptimeStatus = new UptimeStatus();
+    uptimeStatus.setFromDocumentString(RELAYS_AND_BRIDGES_UPTIME_SAMPLE);
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2013-09-20 12:00:00"), RUNNING_FLAG);
+    uptimeStatus.compressHistory();
+    assertEquals("Mixed relay history must not be compressed.", 3,
+        uptimeStatus.getRelayHistory().size());
+  }
+
+  private static final String RELAYS_FLAGS_UPTIME_SAMPLE =
+      "R 2013-07-22-17 5811 Running\n"; /* ends 2014-03-21 20:00:00 */
+
+  @Test()
+  public void testAddFlagsToFlagsEnd() {
+    UptimeStatus uptimeStatus = new UptimeStatus();
+    uptimeStatus.setFromDocumentString(RELAYS_FLAGS_UPTIME_SAMPLE);
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2014-03-21 20:00:00"), RUNNING_FLAG);
+    uptimeStatus.compressHistory();
+    assertEquals("Relay history with flags must be compressed.", 1,
+        uptimeStatus.getRelayHistory().size());
+  }
+
+  private static final SortedSet<String> RUNNING_VALID_FLAGS =
+      new TreeSet<String>(Arrays.asList(new String[] { "Running",
+      "Valid" }));
+
+  @Test()
+  public void testDontCompressDifferentFlags() {
+    UptimeStatus uptimeStatus = new UptimeStatus();
+    uptimeStatus.setFromDocumentString(RELAYS_FLAGS_UPTIME_SAMPLE);
+    uptimeStatus.addToHistory(true,
+        DateTimeHelper.parse("2014-03-21 20:00:00"), RUNNING_VALID_FLAGS);
+    uptimeStatus.compressHistory();
+    assertEquals("Relay history with different flags must not be "
+        + "compressed.", 2, uptimeStatus.getRelayHistory().size());
+  }
 }
 
diff --git a/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java b/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java
index 8f164a0..a2d1e3e 100644
--- a/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java
+++ b/src/test/java/org/torproject/onionoo/updater/DummyConsensus.java
@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.TreeMap;
+import java.util.TreeSet;
 
 import org.torproject.descriptor.DirSourceEntry;
 import org.torproject.descriptor.DirectorySignature;
@@ -70,8 +71,12 @@ public class DummyConsensus implements RelayNetworkStatusConsensus {
     return null;
   }
 
+  private SortedSet<String> knownFlags = new TreeSet<String>();
+  public void addKnownFlag(String flag) {
+    this.knownFlags.add(flag);
+  }
   public SortedSet<String> getKnownFlags() {
-    return null;
+    return this.knownFlags;
   }
 
   public SortedMap<String, Integer> getConsensusParams() {
diff --git a/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java b/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java
index e74dc5c..25399ab 100644
--- a/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java
+++ b/src/test/java/org/torproject/onionoo/updater/UptimeStatusUpdaterTest.java
@@ -52,6 +52,7 @@ public class UptimeStatusUpdaterTest {
     statusEntry.addFlag("Running");
     DummyConsensus consensus = new DummyConsensus();
     consensus.setValidAfterMillis(VALID_AFTER_SAMPLE);
+    consensus.addKnownFlag("Running");
     consensus.addStatusEntry(statusEntry);
     this.descriptorSource.addDescriptor(DescriptorType.RELAY_CONSENSUSES,
         consensus);
@@ -83,7 +84,7 @@ public class UptimeStatusUpdaterTest {
   private static final String ALL_RELAYS_AND_BRIDGES_FINGERPRINT = null;
 
   private static final String ALL_RELAYS_AND_BRIDGES_UPTIME_SAMPLE =
-      "r 2013-07-22-17 5811\n" /* ends 2014-03-21 20:00:00 */
+      "R 2013-07-22-17 5811 Running\n" /* ends 2014-03-21 20:00:00 */
       + "b 2013-07-22-17 5811\n"; /* ends 2014-03-21 20:00:00 */
 
   private void addAllRelaysAndBridgesUptimeSample() {
@@ -105,7 +106,7 @@ public class UptimeStatusUpdaterTest {
         2, this.documentStore.getPerformedStoreOperations());
     UptimeStatus status = this.documentStore.getDocument(
         UptimeStatus.class, ALL_RELAYS_AND_BRIDGES_FINGERPRINT);
-    assertEquals("Relay history must contain one entry", 1,
+    assertEquals("Relay history must contain one entry.", 1,
         status.getRelayHistory().size());
     UptimeHistory history = status.getRelayHistory().first();
     assertEquals("History not for relay.", true, history.isRelay());
diff --git a/web/protocol.html b/web/protocol.html
index 2a13be8..c1dfa3f 100644
--- a/web/protocol.html
+++ b/web/protocol.html
@@ -174,6 +174,8 @@ field from details documents and optional "advertised_bandwidth" and
 <li><strong>2.2</strong>: Removed optional "pool_assignment" field and
 added "transports" field to bridge details documents on December 8,
 2014.</li>
+<li><strong>2.3</strong>: Added optional "flags" field to uptime
+documents on March 22, 2015.</li>
 </ul>
 
 </div> <!-- box -->
@@ -2206,6 +2208,23 @@ of network statuses have been processed for a given time period.
 </p>
 </li>
 
+<li>
+<b><font color="blue">flags</font></b>
+<code class="typeof">object</code>
+<span class="required-false">optional</span>
+<p>
+Object containing fractional times of this relay having relay flags
+assigned.
+Keys are flag names like <strong>"Running"</strong> or
+<strong>"Exit"</strong>, values are objects similar to the
+<strong>uptime</strong> field above, again with keys like
+<strong>"1_week"</strong> etc.
+If a relay never had a given relay flag assigned, no object is included
+for that flag.
+<font color="blue">Added on March 22, 2015.</font>
+</p>
+</li>
+
 </ul>
 
 <h4>Bridge uptime objects</h4>



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits