[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [onionoo/master] Split node data writer into two classes.



commit e152b6761b125ddcbfe57fc8713043e2af0ad2a3
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Fri Mar 14 15:53:54 2014 +0100

    Split node data writer into two classes.
---
 .../torproject/onionoo/DetailsDocumentWriter.java  |  380 ++++++++
 src/org/torproject/onionoo/Main.java               |   15 +-
 src/org/torproject/onionoo/NodeDataWriter.java     |  959 --------------------
 .../onionoo/NodeDetailsStatusUpdater.java          |  634 +++++++++++++
 4 files changed, 1022 insertions(+), 966 deletions(-)

diff --git a/src/org/torproject/onionoo/DetailsDocumentWriter.java b/src/org/torproject/onionoo/DetailsDocumentWriter.java
new file mode 100644
index 0000000..0fd47c5
--- /dev/null
+++ b/src/org/torproject/onionoo/DetailsDocumentWriter.java
@@ -0,0 +1,380 @@
+package org.torproject.onionoo;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeSet;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.ExitList;
+import org.torproject.descriptor.ExitListEntry;
+
+public class DetailsDocumentWriter implements DescriptorListener,
+    FingerprintListener, DocumentWriter {
+
+  private DescriptorSource descriptorSource;
+
+  private DocumentStore documentStore;
+
+  private long now;
+
+  public DetailsDocumentWriter(DescriptorSource descriptorSource,
+      DocumentStore documentStore, Time time) {
+    this.descriptorSource = descriptorSource;
+    this.documentStore = documentStore;
+    this.now = time.currentTimeMillis();
+    this.registerDescriptorListeners();
+    this.registerFingerprintListeners();
+  }
+
+  private void registerDescriptorListeners() {
+    this.descriptorSource.registerDescriptorListener(this,
+        DescriptorType.EXIT_LISTS);
+  }
+
+  public void processDescriptor(Descriptor descriptor, boolean relay) {
+    if (descriptor instanceof ExitList) {
+      this.processExitList((ExitList) descriptor);
+    }
+  }
+
+  private Map<String, Set<ExitListEntry>> exitListEntries =
+      new HashMap<String, Set<ExitListEntry>>();
+
+  /* TODO Processing descriptors should really be done in
+   * NodeDetailsStatusUpdater, not here.  This is also a bug, because
+   * we're only considering newly published exit lists. */
+  private void processExitList(ExitList exitList) {
+    for (ExitListEntry exitListEntry : exitList.getExitListEntries()) {
+      if (exitListEntry.getScanMillis() <
+          this.now - 24L * 60L * 60L * 1000L) {
+        continue;
+      }
+      String fingerprint = exitListEntry.getFingerprint();
+      if (!this.exitListEntries.containsKey(fingerprint)) {
+        this.exitListEntries.put(fingerprint,
+            new HashSet<ExitListEntry>());
+      }
+      this.exitListEntries.get(fingerprint).add(exitListEntry);
+    }
+  }
+
+  private void registerFingerprintListeners() {
+    this.descriptorSource.registerFingerprintListener(this,
+        DescriptorType.RELAY_CONSENSUSES);
+    this.descriptorSource.registerFingerprintListener(this,
+        DescriptorType.RELAY_SERVER_DESCRIPTORS);
+    this.descriptorSource.registerFingerprintListener(this,
+        DescriptorType.BRIDGE_STATUSES);
+    this.descriptorSource.registerFingerprintListener(this,
+        DescriptorType.BRIDGE_SERVER_DESCRIPTORS);
+    this.descriptorSource.registerFingerprintListener(this,
+        DescriptorType.BRIDGE_POOL_ASSIGNMENTS);
+    this.descriptorSource.registerFingerprintListener(this,
+        DescriptorType.EXIT_LISTS);
+  }
+
+  private SortedSet<String> newRelays = new TreeSet<String>(),
+      newBridges = new TreeSet<String>();
+
+  public void processFingerprints(SortedSet<String> fingerprints,
+      boolean relay) {
+    if (relay) {
+      this.newRelays.addAll(fingerprints);
+    } else {
+      this.newBridges.addAll(fingerprints);
+    }
+  }
+
+  public void writeDocuments() {
+    this.updateRelayDetailsFiles();
+    this.updateBridgeDetailsFiles();
+    Logger.printStatusTime("Wrote details document files");
+  }
+
+  private void updateRelayDetailsFiles() {
+    SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    for (String fingerprint : this.newRelays) {
+
+      /* Generate network-status-specific part. */
+      NodeStatus entry = this.documentStore.retrieve(NodeStatus.class,
+          true, fingerprint);
+      if (entry == null) {
+        continue;
+      }
+      String nickname = entry.getNickname();
+      String address = entry.getAddress();
+      List<String> orAddresses = new ArrayList<String>();
+      orAddresses.add(address + ":" + entry.getOrPort());
+      orAddresses.addAll(entry.getOrAddressesAndPorts());
+      StringBuilder orAddressesAndPortsBuilder = new StringBuilder();
+      int addressesWritten = 0;
+      for (String orAddress : orAddresses) {
+        orAddressesAndPortsBuilder.append(
+            (addressesWritten++ > 0 ? "," : "") + "\""
+            + orAddress.toLowerCase() + "\"");
+      }
+      String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis());
+      String firstSeen = dateTimeFormat.format(
+          entry.getFirstSeenMillis());
+      String lastChangedOrAddress = dateTimeFormat.format(
+          entry.getLastChangedOrAddress());
+      String running = entry.getRunning() ? "true" : "false";
+      int dirPort = entry.getDirPort();
+      String countryCode = entry.getCountryCode();
+      String latitude = entry.getLatitude();
+      String longitude = entry.getLongitude();
+      String countryName = entry.getCountryName();
+      String regionName = entry.getRegionName();
+      String cityName = entry.getCityName();
+      String aSNumber = entry.getASNumber();
+      String aSName = entry.getASName();
+      long consensusWeight = entry.getConsensusWeight();
+      String hostName = entry.getHostName();
+      double advertisedBandwidthFraction =
+          entry.getAdvertisedBandwidthFraction();
+      double consensusWeightFraction = entry.getConsensusWeightFraction();
+      double guardProbability = entry.getGuardProbability();
+      double middleProbability = entry.getMiddleProbability();
+      double exitProbability = entry.getExitProbability();
+      String defaultPolicy = entry.getDefaultPolicy();
+      String portList = entry.getPortList();
+      Boolean recommendedVersion = entry.getRecommendedVersion();
+      StringBuilder sb = new StringBuilder();
+      sb.append("{\"version\":1,\n"
+          + "\"nickname\":\"" + nickname + "\",\n"
+          + "\"fingerprint\":\"" + fingerprint + "\",\n"
+          + "\"or_addresses\":[" + orAddressesAndPortsBuilder.toString()
+          + "]");
+      if (dirPort != 0) {
+        sb.append(",\n\"dir_address\":\"" + address + ":" + dirPort
+            + "\"");
+      }
+      sb.append(",\n\"last_seen\":\"" + lastSeen + "\"");
+      sb.append(",\n\"first_seen\":\"" + firstSeen + "\"");
+      sb.append(",\n\"last_changed_address_or_port\":\""
+          + lastChangedOrAddress + "\"");
+      sb.append(",\n\"running\":" + running);
+      SortedSet<String> relayFlags = entry.getRelayFlags();
+      if (!relayFlags.isEmpty()) {
+        sb.append(",\n\"flags\":[");
+        int written = 0;
+        for (String relayFlag : relayFlags) {
+          sb.append((written++ > 0 ? "," : "") + "\"" + relayFlag + "\"");
+        }
+        sb.append("]");
+      }
+      if (countryCode != null) {
+        sb.append(",\n\"country\":\"" + countryCode + "\"");
+      }
+      if (latitude != null) {
+        sb.append(",\n\"latitude\":" + latitude);
+      }
+      if (longitude != null) {
+        sb.append(",\n\"longitude\":" + longitude);
+      }
+      if (countryName != null) {
+        sb.append(",\n\"country_name\":\""
+            + escapeJSON(countryName) + "\"");
+      }
+      if (regionName != null) {
+        sb.append(",\n\"region_name\":\""
+            + escapeJSON(regionName) + "\"");
+      }
+      if (cityName != null) {
+        sb.append(",\n\"city_name\":\""
+            + escapeJSON(cityName) + "\"");
+      }
+      if (aSNumber != null) {
+        sb.append(",\n\"as_number\":\""
+            + escapeJSON(aSNumber) + "\"");
+      }
+      if (aSName != null) {
+        sb.append(",\n\"as_name\":\""
+            + escapeJSON(aSName) + "\"");
+      }
+      if (consensusWeight >= 0L) {
+        sb.append(",\n\"consensus_weight\":"
+            + String.valueOf(consensusWeight));
+      }
+      if (hostName != null) {
+        sb.append(",\n\"host_name\":\""
+            + escapeJSON(hostName) + "\"");
+      }
+      if (advertisedBandwidthFraction >= 0.0) {
+        sb.append(String.format(
+            ",\n\"advertised_bandwidth_fraction\":%.9f",
+            advertisedBandwidthFraction));
+      }
+      if (consensusWeightFraction >= 0.0) {
+        sb.append(String.format(",\n\"consensus_weight_fraction\":%.9f",
+            consensusWeightFraction));
+      }
+      if (guardProbability >= 0.0) {
+        sb.append(String.format(",\n\"guard_probability\":%.9f",
+            guardProbability));
+      }
+      if (middleProbability >= 0.0) {
+        sb.append(String.format(",\n\"middle_probability\":%.9f",
+            middleProbability));
+      }
+      if (exitProbability >= 0.0) {
+        sb.append(String.format(",\n\"exit_probability\":%.9f",
+            exitProbability));
+      }
+      if (defaultPolicy != null && (defaultPolicy.equals("accept") ||
+          defaultPolicy.equals("reject")) && portList != null) {
+        sb.append(",\n\"exit_policy_summary\":{\"" + defaultPolicy
+            + "\":[");
+        int portsWritten = 0;
+        for (String portOrPortRange : portList.split(",")) {
+          sb.append((portsWritten++ > 0 ? "," : "")
+              + "\"" + portOrPortRange + "\"");
+        }
+        sb.append("]}");
+      }
+      if (recommendedVersion != null) {
+        sb.append(",\n\"recommended_version\":" + (recommendedVersion ?
+            "true" : "false"));
+      }
+
+      /* Add exit addresses if at least one of them is distinct from the
+       * onion-routing addresses. */
+      if (exitListEntries.containsKey(fingerprint)) {
+        for (ExitListEntry exitListEntry :
+            exitListEntries.get(fingerprint)) {
+          entry.addExitAddress(exitListEntry.getExitAddress());
+        }
+      }
+      if (!entry.getExitAddresses().isEmpty()) {
+        sb.append(",\n\"exit_addresses\":[");
+        int written = 0;
+        for (String exitAddress : entry.getExitAddresses()) {
+          sb.append((written++ > 0 ? "," : "") + "\""
+              + exitAddress.toLowerCase() + "\"");
+        }
+        sb.append("]");
+      }
+
+      /* Append descriptor-specific part from details status file, and
+       * update contact in node status. */
+      DetailsStatus detailsStatus = this.documentStore.retrieve(
+          DetailsStatus.class, false, fingerprint);
+      if (detailsStatus != null &&
+          detailsStatus.documentString.length() > 0) {
+        sb.append(",\n" + detailsStatus.documentString);
+        String contact = null;
+        Scanner s = new Scanner(detailsStatus.documentString);
+        while (s.hasNextLine()) {
+          String line = s.nextLine();
+          if (!line.startsWith("\"contact\":")) {
+            continue;
+          }
+          int start = "\"contact\":\"".length(), end = line.length() - 1;
+          if (line.endsWith(",")) {
+            end--;
+          }
+          contact = unescapeJSON(line.substring(start, end));
+          break;
+        }
+        s.close();
+        entry.setContact(contact);
+      }
+
+      /* Finish details string. */
+      sb.append("\n}\n");
+
+      /* Write details file to disk. */
+      DetailsDocument detailsDocument = new DetailsDocument();
+      detailsDocument.documentString = sb.toString();
+      this.documentStore.store(detailsDocument, fingerprint);
+    }
+  }
+
+  private void updateBridgeDetailsFiles() {
+    SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    for (String fingerprint : this.newBridges) {
+
+      /* Generate network-status-specific part. */
+      NodeStatus entry = this.documentStore.retrieve(NodeStatus.class,
+          true, fingerprint);
+      if (entry == null) {
+        continue;
+      }
+      String nickname = entry.getNickname();
+      String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis());
+      String firstSeen = dateTimeFormat.format(
+          entry.getFirstSeenMillis());
+      String running = entry.getRunning() ? "true" : "false";
+      String address = entry.getAddress();
+      List<String> orAddresses = new ArrayList<String>();
+      orAddresses.add(address + ":" + entry.getOrPort());
+      orAddresses.addAll(entry.getOrAddressesAndPorts());
+      StringBuilder orAddressesAndPortsBuilder = new StringBuilder();
+      int addressesWritten = 0;
+      for (String orAddress : orAddresses) {
+        orAddressesAndPortsBuilder.append(
+            (addressesWritten++ > 0 ? "," : "") + "\""
+            + orAddress.toLowerCase() + "\"");
+      }
+      StringBuilder sb = new StringBuilder();
+      sb.append("{\"version\":1,\n"
+          + "\"nickname\":\"" + nickname + "\",\n"
+          + "\"hashed_fingerprint\":\"" + fingerprint + "\",\n"
+          + "\"or_addresses\":[" + orAddressesAndPortsBuilder.toString()
+          + "],\n\"last_seen\":\"" + lastSeen + "\",\n\"first_seen\":\""
+          + firstSeen + "\",\n\"running\":" + running);
+
+      SortedSet<String> relayFlags = entry.getRelayFlags();
+      if (!relayFlags.isEmpty()) {
+        sb.append(",\n\"flags\":[");
+        int written = 0;
+        for (String relayFlag : relayFlags) {
+          sb.append((written++ > 0 ? "," : "") + "\"" + relayFlag + "\"");
+        }
+        sb.append("]");
+      }
+
+      /* Append descriptor-specific part from details status file. */
+      DetailsStatus detailsStatus = this.documentStore.retrieve(
+          DetailsStatus.class, false, fingerprint);
+      if (detailsStatus != null &&
+          detailsStatus.documentString.length() > 0) {
+        sb.append(",\n" + detailsStatus.documentString);
+      }
+
+      /* Finish details string. */
+      sb.append("\n}\n");
+
+      /* Write details file to disk. */
+      DetailsDocument detailsDocument = new DetailsDocument();
+      detailsDocument.documentString = sb.toString();
+      this.documentStore.store(detailsDocument, fingerprint);
+    }
+  }
+
+  private static String escapeJSON(String s) {
+    return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'");
+  }
+
+  private static String unescapeJSON(String s) {
+    return StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'"));
+  }
+
+  public String getStatsString() {
+    /* TODO Add statistics string. */
+    return null;
+  }
+}
diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java
index 60db116..355baac 100644
--- a/src/org/torproject/onionoo/Main.java
+++ b/src/org/torproject/onionoo/Main.java
@@ -30,7 +30,8 @@ public class Main {
     Logger.printStatusTime("Initialized Geoip lookup service");
     ReverseDomainNameResolver rdnr = new ReverseDomainNameResolver(t);
     Logger.printStatusTime("Initialized reverse domain name resolver");
-    NodeDataWriter ndw = new NodeDataWriter(dso, rdnr, ls, ds, t);
+    NodeDetailsStatusUpdater ndsu = new NodeDetailsStatusUpdater(dso,
+        rdnr, ls, ds, t);
     Logger.printStatusTime("Initialized node data writer");
     BandwidthStatusUpdater bsu = new BandwidthStatusUpdater(dso, ds, t);
     Logger.printStatusTime("Initialized bandwidth status updater");
@@ -40,8 +41,11 @@ public class Main {
     Logger.printStatusTime("Initialized clients status updater");
     UptimeStatusUpdater usu = new UptimeStatusUpdater(dso, ds);
     Logger.printStatusTime("Initialized uptime status updater");
-    StatusUpdater[] sus = new StatusUpdater[] { ndw, bsu, wsu, csu, usu };
+    StatusUpdater[] sus = new StatusUpdater[] { ndsu, bsu, wsu, csu,
+        usu };
 
+    DetailsDocumentWriter ddw = new DetailsDocumentWriter(dso, ds, t);
+    Logger.printStatusTime("Initialized details document writer");
     BandwidthDocumentWriter bdw = new BandwidthDocumentWriter(dso, ds, t);
     Logger.printStatusTime("Initialized bandwidth document writer");
     WeightsDocumentWriter wdw = new WeightsDocumentWriter(dso, ds, t);
@@ -50,7 +54,7 @@ public class Main {
     Logger.printStatusTime("Initialized clients document writer");
     UptimeDocumentWriter udw = new UptimeDocumentWriter(dso, ds, t);
     Logger.printStatusTime("Initialized uptime document writer");
-    DocumentWriter[] dws = new DocumentWriter[] { ndw, bdw, wdw, cdw,
+    DocumentWriter[] dws = new DocumentWriter[] { ddw, bdw, wdw, cdw,
         udw };
 
     Logger.printStatus("Reading descriptors.");
@@ -95,10 +99,7 @@ public class Main {
             statsString);
       }
     }
-    /* TODO Print status updater statistics for *all* status updaters once
-     * all data writers have been separated. */
-    for (DocumentWriter dw : new DocumentWriter[] { bdw, wdw, cdw,
-        udw }) {
+    for (DocumentWriter dw : dws) {
       String statsString = dw.getStatsString();
       if (statsString != null) {
         Logger.printStatistics(dw.getClass().getSimpleName(),
diff --git a/src/org/torproject/onionoo/NodeDataWriter.java b/src/org/torproject/onionoo/NodeDataWriter.java
deleted file mode 100644
index 5941b1c..0000000
--- a/src/org/torproject/onionoo/NodeDataWriter.java
+++ /dev/null
@@ -1,959 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Scanner;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.commons.lang.StringEscapeUtils;
-import org.torproject.descriptor.BridgeNetworkStatus;
-import org.torproject.descriptor.BridgePoolAssignment;
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.ExitList;
-import org.torproject.descriptor.ExitListEntry;
-import org.torproject.descriptor.NetworkStatusEntry;
-import org.torproject.descriptor.RelayNetworkStatusConsensus;
-import org.torproject.descriptor.ServerDescriptor;
-import org.torproject.onionoo.LookupService.LookupResult;
-
-/* Write updated summary and details data files to disk.
- *
- * The parts of details files coming from server descriptors always come
- * from the last known descriptor of a relay or bridge, not from the
- * descriptor that was last referenced in a network status. */
-public class NodeDataWriter implements DescriptorListener, StatusUpdater,
-    FingerprintListener, DocumentWriter {
-
-  private DescriptorSource descriptorSource;
-
-  private ReverseDomainNameResolver reverseDomainNameResolver;
-
-  private LookupService lookupService;
-
-  private DocumentStore documentStore;
-
-  private long now;
-
-  private SortedMap<String, NodeStatus> knownNodes =
-      new TreeMap<String, NodeStatus>();
-
-  private SortedMap<String, NodeStatus> relays;
-
-  private SortedMap<String, NodeStatus> bridges;
-
-  private long relaysLastValidAfterMillis = -1L;
-
-  private long bridgesLastPublishedMillis = -1L;
-
-  private SortedMap<String, Integer> lastBandwidthWeights = null;
-
-  private int relayConsensusesProcessed = 0, bridgeStatusesProcessed = 0;
-
-  public NodeDataWriter(DescriptorSource descriptorSource,
-      ReverseDomainNameResolver reverseDomainNameResolver,
-      LookupService lookupService, DocumentStore documentStore,
-      Time time) {
-    this.descriptorSource = descriptorSource;
-    this.reverseDomainNameResolver = reverseDomainNameResolver;
-    this.lookupService = lookupService;
-    this.documentStore = documentStore;
-    this.now = time.currentTimeMillis();
-    this.registerDescriptorListeners();
-    this.registerFingerprintListeners();
-  }
-
-  private void registerDescriptorListeners() {
-    this.descriptorSource.registerDescriptorListener(this,
-        DescriptorType.RELAY_CONSENSUSES);
-    this.descriptorSource.registerDescriptorListener(this,
-        DescriptorType.RELAY_SERVER_DESCRIPTORS);
-    this.descriptorSource.registerDescriptorListener(this,
-        DescriptorType.BRIDGE_STATUSES);
-    this.descriptorSource.registerDescriptorListener(this,
-        DescriptorType.BRIDGE_SERVER_DESCRIPTORS);
-    this.descriptorSource.registerDescriptorListener(this,
-        DescriptorType.BRIDGE_POOL_ASSIGNMENTS);
-    this.descriptorSource.registerDescriptorListener(this,
-        DescriptorType.EXIT_LISTS);
-  }
-
-  private void registerFingerprintListeners() {
-    /* TODO Not used yet.
-    this.descriptorSource.registerFingerprintListener(this,
-        DescriptorType.RELAY_CONSENSUSES);
-    this.descriptorSource.registerFingerprintListener(this,
-        DescriptorType.RELAY_SERVER_DESCRIPTORS);
-    this.descriptorSource.registerFingerprintListener(this,
-        DescriptorType.BRIDGE_STATUSES);
-    this.descriptorSource.registerFingerprintListener(this,
-        DescriptorType.BRIDGE_SERVER_DESCRIPTORS);
-    this.descriptorSource.registerFingerprintListener(this,
-        DescriptorType.BRIDGE_POOL_ASSIGNMENTS);
-    this.descriptorSource.registerFingerprintListener(this,
-        DescriptorType.EXIT_LISTS);*/
-  }
-
-  public void processDescriptor(Descriptor descriptor, boolean relay) {
-    if (descriptor instanceof RelayNetworkStatusConsensus) {
-      this.updateRelayNetworkStatusConsensus(
-          (RelayNetworkStatusConsensus) descriptor);
-    } else if (descriptor instanceof ServerDescriptor && relay) {
-      this.processRelayServerDescriptor((ServerDescriptor) descriptor);
-    } else if (descriptor instanceof BridgeNetworkStatus) {
-      this.updateBridgeNetworkStatus((BridgeNetworkStatus) descriptor);
-    } else if (descriptor instanceof ServerDescriptor && !relay) {
-      this.processBridgeServerDescriptor((ServerDescriptor) descriptor);
-    } else if (descriptor instanceof BridgePoolAssignment) {
-      this.processBridgePoolAssignment((BridgePoolAssignment) descriptor);
-    } else if (descriptor instanceof ExitList) {
-      this.processExitList((ExitList) descriptor);
-    }
-  }
-
-  public void processFingerprints(SortedSet<String> fingerprints,
-      boolean relay) {
-    /* TODO Not used yet. */
-  }
-
-  public void updateStatuses() {
-    this.readStatusSummary();
-    Logger.printStatusTime("Read status summary");
-    this.setCurrentNodes();
-    Logger.printStatusTime("Set current node fingerprints");
-    this.startReverseDomainNameLookups();
-    Logger.printStatusTime("Started reverse domain name lookups");
-    this.lookUpCitiesAndASes();
-    Logger.printStatusTime("Looked up cities and ASes");
-    this.setRunningBits();
-    Logger.printStatusTime("Set running bits");
-    this.calculatePathSelectionProbabilities();
-    Logger.printStatusTime("Calculated path selection probabilities");
-    this.finishReverseDomainNameLookups();
-    Logger.printStatusTime("Finished reverse domain name lookups");
-    this.writeStatusSummary();
-    Logger.printStatusTime("Wrote status summary");
-    this.writeOutSummary();
-    Logger.printStatusTime("Wrote out summary");
-  }
-
-  public void writeDocuments() {
-    this.writeOutDetails();
-    Logger.printStatusTime("Wrote detail data files");
-  }
-
-  private void updateRelayNetworkStatusConsensus(
-      RelayNetworkStatusConsensus consensus) {
-    long validAfterMillis = consensus.getValidAfterMillis();
-    if (validAfterMillis > this.relaysLastValidAfterMillis) {
-      this.relaysLastValidAfterMillis = validAfterMillis;
-    }
-    Set<String> recommendedVersions = null;
-    if (consensus.getRecommendedServerVersions() != null) {
-      recommendedVersions = new HashSet<String>();
-      for (String recommendedVersion :
-          consensus.getRecommendedServerVersions()) {
-        recommendedVersions.add("Tor " + recommendedVersion);
-      }
-    }
-    for (NetworkStatusEntry entry :
-        consensus.getStatusEntries().values()) {
-      String nickname = entry.getNickname();
-      String fingerprint = entry.getFingerprint();
-      String address = entry.getAddress();
-      SortedSet<String> orAddressesAndPorts = new TreeSet<String>(
-          entry.getOrAddresses());
-      int orPort = entry.getOrPort();
-      int dirPort = entry.getDirPort();
-      SortedSet<String> relayFlags = entry.getFlags();
-      long consensusWeight = entry.getBandwidth();
-      String defaultPolicy = entry.getDefaultPolicy();
-      String portList = entry.getPortList();
-      Boolean recommendedVersion = (recommendedVersions == null ||
-          entry.getVersion() == null) ? null :
-          recommendedVersions.contains(entry.getVersion());
-      NodeStatus newNodeStatus = new NodeStatus(true, nickname,
-          fingerprint, address, orAddressesAndPorts, null,
-          validAfterMillis, orPort, dirPort, relayFlags, consensusWeight,
-          null, null, -1L, defaultPolicy, portList, validAfterMillis,
-          validAfterMillis, null, null, recommendedVersion);
-      if (this.knownNodes.containsKey(fingerprint)) {
-        this.knownNodes.get(fingerprint).update(newNodeStatus);
-      } else {
-        this.knownNodes.put(fingerprint, newNodeStatus);
-      }
-    }
-    this.relayConsensusesProcessed++;
-    if (this.relaysLastValidAfterMillis == validAfterMillis) {
-      this.lastBandwidthWeights = consensus.getBandwidthWeights();
-    }
-  }
-
-  private void updateBridgeNetworkStatus(BridgeNetworkStatus status) {
-    long publishedMillis = status.getPublishedMillis();
-    if (publishedMillis > this.bridgesLastPublishedMillis) {
-      this.bridgesLastPublishedMillis = publishedMillis;
-    }
-    for (NetworkStatusEntry entry : status.getStatusEntries().values()) {
-      String nickname = entry.getNickname();
-      String fingerprint = entry.getFingerprint();
-      String address = entry.getAddress();
-      SortedSet<String> orAddressesAndPorts = new TreeSet<String>(
-          entry.getOrAddresses());
-      int orPort = entry.getOrPort();
-      int dirPort = entry.getDirPort();
-      SortedSet<String> relayFlags = entry.getFlags();
-      NodeStatus newNodeStatus = new NodeStatus(false, nickname,
-          fingerprint, address, orAddressesAndPorts, null,
-          publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null,
-          -1L, null, null, publishedMillis, -1L, null, null, null);
-      if (this.knownNodes.containsKey(fingerprint)) {
-        this.knownNodes.get(fingerprint).update(newNodeStatus);
-      } else {
-        this.knownNodes.put(fingerprint, newNodeStatus);
-      }
-    }
-    this.bridgeStatusesProcessed++;
-  }
-
-  private void readStatusSummary() {
-    SortedSet<String> fingerprints = this.documentStore.list(
-        NodeStatus.class, true);
-    for (String fingerprint : fingerprints) {
-      NodeStatus node = this.documentStore.retrieve(NodeStatus.class,
-          true, fingerprint);
-      if (node.isRelay()) {
-        this.relaysLastValidAfterMillis = Math.max(
-            this.relaysLastValidAfterMillis, node.getLastSeenMillis());
-      } else {
-        this.bridgesLastPublishedMillis = Math.max(
-            this.bridgesLastPublishedMillis, node.getLastSeenMillis());
-      }
-      if (this.knownNodes.containsKey(fingerprint)) {
-        this.knownNodes.get(fingerprint).update(node);
-      } else {
-        this.knownNodes.put(fingerprint, node);
-      }
-    }
-  }
-
-  private void setRunningBits() {
-    for (NodeStatus node : this.knownNodes.values()) {
-      if (node.isRelay() && node.getRelayFlags().contains("Running") &&
-          node.getLastSeenMillis() == this.relaysLastValidAfterMillis) {
-        node.setRunning(true);
-      }
-      if (!node.isRelay() && node.getRelayFlags().contains("Running") &&
-          node.getLastSeenMillis() == this.bridgesLastPublishedMillis) {
-        node.setRunning(true);
-      }
-    }
-  }
-
-  private void lookUpCitiesAndASes() {
-    SortedSet<String> addressStrings = new TreeSet<String>();
-    for (NodeStatus node : this.knownNodes.values()) {
-      if (node.isRelay()) {
-        addressStrings.add(node.getAddress());
-      }
-    }
-    if (addressStrings.isEmpty()) {
-      System.err.println("No relay IP addresses to resolve to cities or "
-          + "ASN.");
-      return;
-    }
-    SortedMap<String, LookupResult> lookupResults =
-        this.lookupService.lookup(addressStrings);
-    for (NodeStatus node : knownNodes.values()) {
-      if (!node.isRelay()) {
-        continue;
-      }
-      String addressString = node.getAddress();
-      if (lookupResults.containsKey(addressString)) {
-        LookupResult lookupResult = lookupResults.get(addressString);
-        node.setCountryCode(lookupResult.countryCode);
-        node.setCountryName(lookupResult.countryName);
-        node.setRegionName(lookupResult.regionName);
-        node.setCityName(lookupResult.cityName);
-        node.setLatitude(lookupResult.latitude);
-        node.setLongitude(lookupResult.longitude);
-        node.setASNumber(lookupResult.aSNumber);
-        node.setASName(lookupResult.aSName);
-      }
-    }
-  }
-
-  private void writeStatusSummary() {
-    this.writeSummary(true);
-  }
-
-  private void writeOutSummary() {
-    this.writeSummary(false);
-  }
-
-  private void writeSummary(boolean includeArchive) {
-    SortedMap<String, NodeStatus> nodes = includeArchive
-        ? this.knownNodes : this.getCurrentNodes();
-    for (Map.Entry<String, NodeStatus> e : nodes.entrySet()) {
-      this.documentStore.store(e.getValue(), e.getKey());
-    }
-  }
-
-  private SortedMap<String, NodeStatus> getCurrentNodes() {
-    long cutoff = Math.max(this.relaysLastValidAfterMillis,
-        this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L;
-    SortedMap<String, NodeStatus> currentNodes =
-        new TreeMap<String, NodeStatus>();
-    for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) {
-      if (e.getValue().getLastSeenMillis() >= cutoff) {
-        currentNodes.put(e.getKey(), e.getValue());
-      }
-    }
-    return currentNodes;
-  }
-
-  private void processRelayServerDescriptor(
-      ServerDescriptor descriptor) {
-    String fingerprint = descriptor.getFingerprint();
-    DetailsStatus detailsStatus = this.documentStore.retrieve(
-        DetailsStatus.class, false, fingerprint);
-    SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
-        "yyyy-MM-dd HH:mm:ss");
-    dateTimeFormat.setLenient(false);
-    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    String publishedDateTime =
-        dateTimeFormat.format(descriptor.getPublishedMillis());
-    if (detailsStatus != null) {
-      String detailsString = detailsStatus.documentString;
-      String descPublishedLine = "\"desc_published\":\""
-          + publishedDateTime + "\",";
-      Scanner s = new Scanner(detailsString);
-      while (s.hasNextLine()) {
-        String line = s.nextLine();
-        if (line.startsWith("\"desc_published\":\"")) {
-          if (descPublishedLine.compareTo(line) < 0) {
-            return;
-          } else {
-            break;
-          }
-        }
-      }
-      s.close();
-    }
-    StringBuilder sb = new StringBuilder();
-    String lastRestartedString = dateTimeFormat.format(
-        descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L);
-    int bandwidthRate = descriptor.getBandwidthRate();
-    int bandwidthBurst = descriptor.getBandwidthBurst();
-    int observedBandwidth = descriptor.getBandwidthObserved();
-    int advertisedBandwidth = Math.min(bandwidthRate,
-        Math.min(bandwidthBurst, observedBandwidth));
-    sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n"
-        + "\"last_restarted\":\"" + lastRestartedString + "\",\n"
-        + "\"bandwidth_rate\":" + bandwidthRate + ",\n"
-        + "\"bandwidth_burst\":" + bandwidthBurst + ",\n"
-        + "\"observed_bandwidth\":" + observedBandwidth + ",\n"
-        + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n"
-        + "\"exit_policy\":[");
-    int written = 0;
-    for (String exitPolicyLine : descriptor.getExitPolicyLines()) {
-      sb.append((written++ > 0 ? "," : "") + "\n  \"" + exitPolicyLine
-          + "\"");
-    }
-    sb.append("\n]");
-    if (descriptor.getContact() != null) {
-      sb.append(",\n\"contact\":\""
-          + escapeJSON(descriptor.getContact()) + "\"");
-    }
-    if (descriptor.getPlatform() != null) {
-      sb.append(",\n\"platform\":\""
-          + escapeJSON(descriptor.getPlatform()) + "\"");
-    }
-    if (descriptor.getFamilyEntries() != null) {
-      sb.append(",\n\"family\":[");
-      written = 0;
-      for (String familyEntry : descriptor.getFamilyEntries()) {
-        sb.append((written++ > 0 ? "," : "") + "\n  \"" + familyEntry
-            + "\"");
-      }
-      sb.append("\n]");
-    }
-    if (descriptor.getIpv6DefaultPolicy() != null &&
-        (descriptor.getIpv6DefaultPolicy().equals("accept") ||
-        descriptor.getIpv6DefaultPolicy().equals("reject")) &&
-        descriptor.getIpv6PortList() != null) {
-      sb.append(",\n\"exit_policy_v6_summary\":{\""
-          + descriptor.getIpv6DefaultPolicy() + "\":[");
-      int portsWritten = 0;
-      for (String portOrPortRange :
-          descriptor.getIpv6PortList().split(",")) {
-        sb.append((portsWritten++ > 0 ? "," : "") + "\"" + portOrPortRange
-            + "\"");
-      }
-      sb.append("]}");
-    }
-    if (descriptor.isHibernating()) {
-      sb.append(",\n\"hibernating\":true");
-    }
-    detailsStatus = new DetailsStatus();
-    detailsStatus.documentString = sb.toString();
-    this.documentStore.store(detailsStatus, fingerprint);
-  }
-
-  private void processBridgeServerDescriptor(
-      ServerDescriptor descriptor) {
-    String fingerprint = descriptor.getFingerprint();
-    DetailsStatus detailsStatus = this.documentStore.retrieve(
-        DetailsStatus.class, false, fingerprint);
-    SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
-        "yyyy-MM-dd HH:mm:ss");
-    dateTimeFormat.setLenient(false);
-    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    String publishedDateTime =
-        dateTimeFormat.format(descriptor.getPublishedMillis());
-    String poolAssignmentLine = null;
-    if (detailsStatus != null) {
-      String detailsString = detailsStatus.documentString;
-      String descPublishedLine = "\"desc_published\":\""
-          + publishedDateTime + "\",";
-      Scanner s = new Scanner(detailsString);
-      while (s.hasNextLine()) {
-        String line = s.nextLine();
-        if (line.startsWith("\"pool_assignment\":")) {
-          poolAssignmentLine = line;
-        } else if (line.startsWith("\"desc_published\":") &&
-            descPublishedLine.compareTo(line) < 0) {
-          return;
-        }
-      }
-      s.close();
-    }
-    StringBuilder sb = new StringBuilder();
-    String lastRestartedString = dateTimeFormat.format(
-        descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L);
-    int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(),
-        Math.min(descriptor.getBandwidthBurst(),
-        descriptor.getBandwidthObserved()));
-    sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n"
-        + "\"last_restarted\":\"" + lastRestartedString + "\",\n"
-        + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n"
-        + "\"platform\":\"" + escapeJSON(descriptor.getPlatform())
-        + "\"");
-    if (poolAssignmentLine != null) {
-      sb.append(",\n" + poolAssignmentLine);
-    }
-    detailsStatus = new DetailsStatus();
-    detailsStatus.documentString = sb.toString();
-    this.documentStore.store(detailsStatus, fingerprint);
-  }
-
-  private void processBridgePoolAssignment(
-      BridgePoolAssignment bridgePoolAssignment) {
-    for (Map.Entry<String, String> e :
-        bridgePoolAssignment.getEntries().entrySet()) {
-      String fingerprint = e.getKey();
-      String details = e.getValue();
-      StringBuilder sb = new StringBuilder();
-      DetailsStatus detailsStatus = this.documentStore.retrieve(
-          DetailsStatus.class, false, fingerprint);
-      if (detailsStatus != null) {
-        String detailsString = detailsStatus.documentString;
-        Scanner s = new Scanner(detailsString);
-        int linesWritten = 0;
-        boolean endsWithComma = false;
-        while (s.hasNextLine()) {
-          String line = s.nextLine();
-          if (!line.startsWith("\"pool_assignment\":")) {
-            sb.append((linesWritten++ > 0 ? "\n" : "") + line);
-            endsWithComma = line.endsWith(",");
-          }
-        }
-        s.close();
-        if (sb.length() > 0) {
-          sb.append((endsWithComma ? "" : ",") + "\n");
-        }
-      }
-      sb.append("\"pool_assignment\":\"" + details + "\"");
-      detailsStatus = new DetailsStatus();
-      detailsStatus.documentString = sb.toString();
-      this.documentStore.store(detailsStatus, fingerprint);
-    }
-  }
-
-  private Map<String, Set<ExitListEntry>> exitListEntries =
-      new HashMap<String, Set<ExitListEntry>>();
-
-  private void processExitList(ExitList exitList) {
-    for (ExitListEntry exitListEntry : exitList.getExitListEntries()) {
-      if (exitListEntry.getScanMillis() <
-          this.now - 24L * 60L * 60L * 1000L) {
-        continue;
-      }
-      String fingerprint = exitListEntry.getFingerprint();
-      if (!this.exitListEntries.containsKey(fingerprint)) {
-        this.exitListEntries.put(fingerprint,
-            new HashSet<ExitListEntry>());
-      }
-      this.exitListEntries.get(fingerprint).add(exitListEntry);
-    }
-  }
-
-  private void setCurrentNodes() {
-    SortedMap<String, NodeStatus> currentNodes = this.getCurrentNodes();
-    this.relays = new TreeMap<String, NodeStatus>();
-    this.bridges = new TreeMap<String, NodeStatus>();
-    for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) {
-      if (e.getValue().isRelay()) {
-        this.relays.put(e.getKey(), e.getValue());
-      } else {
-        this.bridges.put(e.getKey(), e.getValue());
-      }
-    }
-  }
-
-  private void startReverseDomainNameLookups() {
-    Map<String, Long> addressLastLookupTimes =
-        new HashMap<String, Long>();
-    for (NodeStatus relay : relays.values()) {
-      addressLastLookupTimes.put(relay.getAddress(),
-          relay.getLastRdnsLookup());
-    }
-    this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes);
-    this.reverseDomainNameResolver.startReverseDomainNameLookups();
-  }
-
-  private void finishReverseDomainNameLookups() {
-    this.reverseDomainNameResolver.finishReverseDomainNameLookups();
-    Map<String, String> lookupResults =
-        this.reverseDomainNameResolver.getLookupResults();
-    long startedRdnsLookups =
-        this.reverseDomainNameResolver.getLookupStartMillis();
-    for (NodeStatus relay : relays.values()) {
-      if (lookupResults.containsKey(relay.getAddress())) {
-        relay.setHostName(lookupResults.get(relay.getAddress()));
-        relay.setLastRdnsLookup(startedRdnsLookups);
-      }
-    }
-  }
-
-  private void calculatePathSelectionProbabilities() {
-    boolean consensusContainsBandwidthWeights = false;
-    double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0,
-        wmd = 0.0, wee = 0.0, wed = 0.0;
-    if (this.lastBandwidthWeights != null) {
-      SortedSet<String> weightKeys = new TreeSet<String>(Arrays.asList(
-          "Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(",")));
-      weightKeys.removeAll(this.lastBandwidthWeights.keySet());
-      if (weightKeys.isEmpty()) {
-        consensusContainsBandwidthWeights = true;
-        wgg = ((double) this.lastBandwidthWeights.get("Wgg")) / 10000.0;
-        wgd = ((double) this.lastBandwidthWeights.get("Wgd")) / 10000.0;
-        wmg = ((double) this.lastBandwidthWeights.get("Wmg")) / 10000.0;
-        wmm = ((double) this.lastBandwidthWeights.get("Wmm")) / 10000.0;
-        wme = ((double) this.lastBandwidthWeights.get("Wme")) / 10000.0;
-        wmd = ((double) this.lastBandwidthWeights.get("Wmd")) / 10000.0;
-        wee = ((double) this.lastBandwidthWeights.get("Wee")) / 10000.0;
-        wed = ((double) this.lastBandwidthWeights.get("Wed")) / 10000.0;
-      }
-    } else {
-      System.err.println("Could not determine most recent Wxx parameter "
-          + "values, probably because we didn't parse a consensus in "
-          + "this execution.  All relays' guard/middle/exit weights are "
-          + "going to be 0.0.");
-    }
-    SortedMap<String, Double>
-        advertisedBandwidths = new TreeMap<String, Double>(),
-        consensusWeights = new TreeMap<String, Double>(),
-        guardWeights = new TreeMap<String, Double>(),
-        middleWeights = new TreeMap<String, Double>(),
-        exitWeights = new TreeMap<String, Double>();
-    double totalAdvertisedBandwidth = 0.0;
-    double totalConsensusWeight = 0.0;
-    double totalGuardWeight = 0.0;
-    double totalMiddleWeight = 0.0;
-    double totalExitWeight = 0.0;
-    for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) {
-      String fingerprint = e.getKey();
-      NodeStatus relay = e.getValue();
-      if (!relay.getRunning()) {
-        continue;
-      }
-      boolean isExit = relay.getRelayFlags().contains("Exit") &&
-          !relay.getRelayFlags().contains("BadExit");
-      boolean isGuard = relay.getRelayFlags().contains("Guard");
-      DetailsStatus detailsStatus = this.documentStore.retrieve(
-          DetailsStatus.class, false, fingerprint);
-      if (detailsStatus != null) {
-        double advertisedBandwidth = -1.0;
-        String detailsString = detailsStatus.documentString;
-        Scanner s = new Scanner(detailsString);
-        while (s.hasNextLine()) {
-          String line = s.nextLine();
-          if (!line.startsWith("\"advertised_bandwidth\":")) {
-            continue;
-          }
-          try {
-            advertisedBandwidth = (double) Integer.parseInt(
-                line.split(":")[1].replaceAll(",", ""));
-          } catch (NumberFormatException ex) {
-            /* Handle below. */
-          }
-          break;
-        }
-        s.close();
-        if (advertisedBandwidth >= 0.0) {
-          advertisedBandwidths.put(fingerprint, advertisedBandwidth);
-          totalAdvertisedBandwidth += advertisedBandwidth;
-        }
-      }
-      double consensusWeight = (double) relay.getConsensusWeight();
-      consensusWeights.put(fingerprint, consensusWeight);
-      totalConsensusWeight += consensusWeight;
-      if (consensusContainsBandwidthWeights) {
-        double guardWeight = consensusWeight,
-            middleWeight = consensusWeight,
-            exitWeight = consensusWeight;
-        if (isGuard && isExit) {
-          guardWeight *= wgd;
-          middleWeight *= wmd;
-          exitWeight *= wed;
-        } else if (isGuard) {
-          guardWeight *= wgg;
-          middleWeight *= wmg;
-          exitWeight = 0.0;
-        } else if (isExit) {
-          guardWeight = 0.0;
-          middleWeight *= wme;
-          exitWeight *= wee;
-        } else {
-          guardWeight = 0.0;
-          middleWeight *= wmm;
-          exitWeight = 0.0;
-        }
-        guardWeights.put(fingerprint, guardWeight);
-        middleWeights.put(fingerprint, middleWeight);
-        exitWeights.put(fingerprint, exitWeight);
-        totalGuardWeight += guardWeight;
-        totalMiddleWeight += middleWeight;
-        totalExitWeight += exitWeight;
-      }
-    }
-    for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) {
-      String fingerprint = e.getKey();
-      NodeStatus relay = e.getValue();
-      if (advertisedBandwidths.containsKey(fingerprint)) {
-        relay.setAdvertisedBandwidthFraction(advertisedBandwidths.get(
-            fingerprint) / totalAdvertisedBandwidth);
-      }
-      if (consensusWeights.containsKey(fingerprint)) {
-        relay.setConsensusWeightFraction(consensusWeights.get(fingerprint)
-            / totalConsensusWeight);
-      }
-      if (guardWeights.containsKey(fingerprint)) {
-        relay.setGuardProbability(guardWeights.get(fingerprint)
-            / totalGuardWeight);
-      }
-      if (middleWeights.containsKey(fingerprint)) {
-        relay.setMiddleProbability(middleWeights.get(fingerprint)
-            / totalMiddleWeight);
-      }
-      if (exitWeights.containsKey(fingerprint)) {
-        relay.setExitProbability(exitWeights.get(fingerprint)
-            / totalExitWeight);
-      }
-    }
-  }
-
-  private void writeOutDetails() {
-    this.updateRelayDetailsFiles();
-    this.updateBridgeDetailsFiles();
-  }
-
-  private static String escapeJSON(String s) {
-    return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'");
-  }
-
-  private static String unescapeJSON(String s) {
-    return StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'"));
-  }
-
-  private void updateRelayDetailsFiles() {
-    SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
-        "yyyy-MM-dd HH:mm:ss");
-    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    for (Map.Entry<String, NodeStatus> relay : this.relays.entrySet()) {
-      String fingerprint = relay.getKey();
-
-      /* Generate network-status-specific part. */
-      NodeStatus entry = relay.getValue();
-      String nickname = entry.getNickname();
-      String address = entry.getAddress();
-      List<String> orAddresses = new ArrayList<String>();
-      orAddresses.add(address + ":" + entry.getOrPort());
-      orAddresses.addAll(entry.getOrAddressesAndPorts());
-      StringBuilder orAddressesAndPortsBuilder = new StringBuilder();
-      int addressesWritten = 0;
-      for (String orAddress : orAddresses) {
-        orAddressesAndPortsBuilder.append(
-            (addressesWritten++ > 0 ? "," : "") + "\""
-            + orAddress.toLowerCase() + "\"");
-      }
-      String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis());
-      String firstSeen = dateTimeFormat.format(
-          entry.getFirstSeenMillis());
-      String lastChangedOrAddress = dateTimeFormat.format(
-          entry.getLastChangedOrAddress());
-      String running = entry.getRunning() ? "true" : "false";
-      int dirPort = entry.getDirPort();
-      String countryCode = entry.getCountryCode();
-      String latitude = entry.getLatitude();
-      String longitude = entry.getLongitude();
-      String countryName = entry.getCountryName();
-      String regionName = entry.getRegionName();
-      String cityName = entry.getCityName();
-      String aSNumber = entry.getASNumber();
-      String aSName = entry.getASName();
-      long consensusWeight = entry.getConsensusWeight();
-      String hostName = entry.getHostName();
-      double advertisedBandwidthFraction =
-          entry.getAdvertisedBandwidthFraction();
-      double consensusWeightFraction = entry.getConsensusWeightFraction();
-      double guardProbability = entry.getGuardProbability();
-      double middleProbability = entry.getMiddleProbability();
-      double exitProbability = entry.getExitProbability();
-      String defaultPolicy = entry.getDefaultPolicy();
-      String portList = entry.getPortList();
-      Boolean recommendedVersion = entry.getRecommendedVersion();
-      StringBuilder sb = new StringBuilder();
-      sb.append("{\"version\":1,\n"
-          + "\"nickname\":\"" + nickname + "\",\n"
-          + "\"fingerprint\":\"" + fingerprint + "\",\n"
-          + "\"or_addresses\":[" + orAddressesAndPortsBuilder.toString()
-          + "]");
-      if (dirPort != 0) {
-        sb.append(",\n\"dir_address\":\"" + address + ":" + dirPort
-            + "\"");
-      }
-      sb.append(",\n\"last_seen\":\"" + lastSeen + "\"");
-      sb.append(",\n\"first_seen\":\"" + firstSeen + "\"");
-      sb.append(",\n\"last_changed_address_or_port\":\""
-          + lastChangedOrAddress + "\"");
-      sb.append(",\n\"running\":" + running);
-      SortedSet<String> relayFlags = entry.getRelayFlags();
-      if (!relayFlags.isEmpty()) {
-        sb.append(",\n\"flags\":[");
-        int written = 0;
-        for (String relayFlag : relayFlags) {
-          sb.append((written++ > 0 ? "," : "") + "\"" + relayFlag + "\"");
-        }
-        sb.append("]");
-      }
-      if (countryCode != null) {
-        sb.append(",\n\"country\":\"" + countryCode + "\"");
-      }
-      if (latitude != null) {
-        sb.append(",\n\"latitude\":" + latitude);
-      }
-      if (longitude != null) {
-        sb.append(",\n\"longitude\":" + longitude);
-      }
-      if (countryName != null) {
-        sb.append(",\n\"country_name\":\""
-            + escapeJSON(countryName) + "\"");
-      }
-      if (regionName != null) {
-        sb.append(",\n\"region_name\":\""
-            + escapeJSON(regionName) + "\"");
-      }
-      if (cityName != null) {
-        sb.append(",\n\"city_name\":\""
-            + escapeJSON(cityName) + "\"");
-      }
-      if (aSNumber != null) {
-        sb.append(",\n\"as_number\":\""
-            + escapeJSON(aSNumber) + "\"");
-      }
-      if (aSName != null) {
-        sb.append(",\n\"as_name\":\""
-            + escapeJSON(aSName) + "\"");
-      }
-      if (consensusWeight >= 0L) {
-        sb.append(",\n\"consensus_weight\":"
-            + String.valueOf(consensusWeight));
-      }
-      if (hostName != null) {
-        sb.append(",\n\"host_name\":\""
-            + escapeJSON(hostName) + "\"");
-      }
-      if (advertisedBandwidthFraction >= 0.0) {
-        sb.append(String.format(
-            ",\n\"advertised_bandwidth_fraction\":%.9f",
-            advertisedBandwidthFraction));
-      }
-      if (consensusWeightFraction >= 0.0) {
-        sb.append(String.format(",\n\"consensus_weight_fraction\":%.9f",
-            consensusWeightFraction));
-      }
-      if (guardProbability >= 0.0) {
-        sb.append(String.format(",\n\"guard_probability\":%.9f",
-            guardProbability));
-      }
-      if (middleProbability >= 0.0) {
-        sb.append(String.format(",\n\"middle_probability\":%.9f",
-            middleProbability));
-      }
-      if (exitProbability >= 0.0) {
-        sb.append(String.format(",\n\"exit_probability\":%.9f",
-            exitProbability));
-      }
-      if (defaultPolicy != null && (defaultPolicy.equals("accept") ||
-          defaultPolicy.equals("reject")) && portList != null) {
-        sb.append(",\n\"exit_policy_summary\":{\"" + defaultPolicy
-            + "\":[");
-        int portsWritten = 0;
-        for (String portOrPortRange : portList.split(",")) {
-          sb.append((portsWritten++ > 0 ? "," : "")
-              + "\"" + portOrPortRange + "\"");
-        }
-        sb.append("]}");
-      }
-      if (recommendedVersion != null) {
-        sb.append(",\n\"recommended_version\":" + (recommendedVersion ?
-            "true" : "false"));
-      }
-
-      /* Add exit addresses if at least one of them is distinct from the
-       * onion-routing addresses. */
-      if (exitListEntries.containsKey(fingerprint)) {
-        for (ExitListEntry exitListEntry :
-            exitListEntries.get(fingerprint)) {
-          entry.addExitAddress(exitListEntry.getExitAddress());
-        }
-      }
-      if (!entry.getExitAddresses().isEmpty()) {
-        sb.append(",\n\"exit_addresses\":[");
-        int written = 0;
-        for (String exitAddress : entry.getExitAddresses()) {
-          sb.append((written++ > 0 ? "," : "") + "\""
-              + exitAddress.toLowerCase() + "\"");
-        }
-        sb.append("]");
-      }
-
-      /* Append descriptor-specific part from details status file, and
-       * update contact in node status. */
-      DetailsStatus detailsStatus = this.documentStore.retrieve(
-          DetailsStatus.class, false, fingerprint);
-      if (detailsStatus != null &&
-          detailsStatus.documentString.length() > 0) {
-        sb.append(",\n" + detailsStatus.documentString);
-        String contact = null;
-        Scanner s = new Scanner(detailsStatus.documentString);
-        while (s.hasNextLine()) {
-          String line = s.nextLine();
-          if (!line.startsWith("\"contact\":")) {
-            continue;
-          }
-          int start = "\"contact\":\"".length(), end = line.length() - 1;
-          if (line.endsWith(",")) {
-            end--;
-          }
-          contact = unescapeJSON(line.substring(start, end));
-          break;
-        }
-        s.close();
-        entry.setContact(contact);
-      }
-
-      /* Finish details string. */
-      sb.append("\n}\n");
-
-      /* Write details file to disk. */
-      DetailsDocument detailsDocument = new DetailsDocument();
-      detailsDocument.documentString = sb.toString();
-      this.documentStore.store(detailsDocument, fingerprint);
-    }
-  }
-
-  private void updateBridgeDetailsFiles() {
-    SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
-        "yyyy-MM-dd HH:mm:ss");
-    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    for (Map.Entry<String, NodeStatus> bridge : this.bridges.entrySet()) {
-      String fingerprint = bridge.getKey();
-
-      /* Generate network-status-specific part. */
-      NodeStatus entry = bridge.getValue();
-      String nickname = entry.getNickname();
-      String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis());
-      String firstSeen = dateTimeFormat.format(
-          entry.getFirstSeenMillis());
-      String running = entry.getRunning() ? "true" : "false";
-      String address = entry.getAddress();
-      List<String> orAddresses = new ArrayList<String>();
-      orAddresses.add(address + ":" + entry.getOrPort());
-      orAddresses.addAll(entry.getOrAddressesAndPorts());
-      StringBuilder orAddressesAndPortsBuilder = new StringBuilder();
-      int addressesWritten = 0;
-      for (String orAddress : orAddresses) {
-        orAddressesAndPortsBuilder.append(
-            (addressesWritten++ > 0 ? "," : "") + "\""
-            + orAddress.toLowerCase() + "\"");
-      }
-      StringBuilder sb = new StringBuilder();
-      sb.append("{\"version\":1,\n"
-          + "\"nickname\":\"" + nickname + "\",\n"
-          + "\"hashed_fingerprint\":\"" + fingerprint + "\",\n"
-          + "\"or_addresses\":[" + orAddressesAndPortsBuilder.toString()
-          + "],\n\"last_seen\":\"" + lastSeen + "\",\n\"first_seen\":\""
-          + firstSeen + "\",\n\"running\":" + running);
-
-      SortedSet<String> relayFlags = entry.getRelayFlags();
-      if (!relayFlags.isEmpty()) {
-        sb.append(",\n\"flags\":[");
-        int written = 0;
-        for (String relayFlag : relayFlags) {
-          sb.append((written++ > 0 ? "," : "") + "\"" + relayFlag + "\"");
-        }
-        sb.append("]");
-      }
-
-      /* Append descriptor-specific part from details status file. */
-      DetailsStatus detailsStatus = this.documentStore.retrieve(
-          DetailsStatus.class, false, fingerprint);
-      if (detailsStatus != null &&
-          detailsStatus.documentString.length() > 0) {
-        sb.append(",\n" + detailsStatus.documentString);
-      }
-
-      /* Finish details string. */
-      sb.append("\n}\n");
-
-      /* Write details file to disk. */
-      DetailsDocument detailsDocument = new DetailsDocument();
-      detailsDocument.documentString = sb.toString();
-      this.documentStore.store(detailsDocument, fingerprint);
-    }
-  }
-
-  public String getStatsString() {
-    StringBuilder sb = new StringBuilder();
-    sb.append("    " + Logger.formatDecimalNumber(
-        relayConsensusesProcessed) + " relay consensuses processed\n");
-    sb.append("    " + Logger.formatDecimalNumber(bridgeStatusesProcessed)
-        + " bridge statuses processed\n");
-    return sb.toString();
-  }
-}
-
diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
new file mode 100644
index 0000000..3c67aea
--- /dev/null
+++ b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
@@ -0,0 +1,634 @@
+/* Copyright 2011, 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo;
+
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.torproject.descriptor.BridgeNetworkStatus;
+import org.torproject.descriptor.BridgePoolAssignment;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.NetworkStatusEntry;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import org.torproject.descriptor.ServerDescriptor;
+import org.torproject.onionoo.LookupService.LookupResult;
+
+public class NodeDetailsStatusUpdater implements DescriptorListener,
+    StatusUpdater {
+
+  private DescriptorSource descriptorSource;
+
+  private ReverseDomainNameResolver reverseDomainNameResolver;
+
+  private LookupService lookupService;
+
+  private DocumentStore documentStore;
+
+  private long now;
+
+  private SortedMap<String, NodeStatus> knownNodes =
+      new TreeMap<String, NodeStatus>();
+
+  private SortedMap<String, NodeStatus> relays;
+
+  private SortedMap<String, NodeStatus> bridges;
+
+  private long relaysLastValidAfterMillis = -1L;
+
+  private long bridgesLastPublishedMillis = -1L;
+
+  private SortedMap<String, Integer> lastBandwidthWeights = null;
+
+  private int relayConsensusesProcessed = 0, bridgeStatusesProcessed = 0;
+
+  public NodeDetailsStatusUpdater(DescriptorSource descriptorSource,
+      ReverseDomainNameResolver reverseDomainNameResolver,
+      LookupService lookupService, DocumentStore documentStore,
+      Time time) {
+    this.descriptorSource = descriptorSource;
+    this.reverseDomainNameResolver = reverseDomainNameResolver;
+    this.lookupService = lookupService;
+    this.documentStore = documentStore;
+    this.now = time.currentTimeMillis();
+    this.registerDescriptorListeners();
+  }
+
+  private void registerDescriptorListeners() {
+    this.descriptorSource.registerDescriptorListener(this,
+        DescriptorType.RELAY_CONSENSUSES);
+    this.descriptorSource.registerDescriptorListener(this,
+        DescriptorType.RELAY_SERVER_DESCRIPTORS);
+    this.descriptorSource.registerDescriptorListener(this,
+        DescriptorType.BRIDGE_STATUSES);
+    this.descriptorSource.registerDescriptorListener(this,
+        DescriptorType.BRIDGE_SERVER_DESCRIPTORS);
+    this.descriptorSource.registerDescriptorListener(this,
+        DescriptorType.BRIDGE_POOL_ASSIGNMENTS);
+    this.descriptorSource.registerDescriptorListener(this,
+        DescriptorType.EXIT_LISTS);
+  }
+
+  public void processDescriptor(Descriptor descriptor, boolean relay) {
+    if (descriptor instanceof RelayNetworkStatusConsensus) {
+      this.processRelayNetworkStatusConsensus(
+          (RelayNetworkStatusConsensus) descriptor);
+    } else if (descriptor instanceof ServerDescriptor && relay) {
+      this.processRelayServerDescriptor((ServerDescriptor) descriptor);
+    } else if (descriptor instanceof BridgeNetworkStatus) {
+      this.processBridgeNetworkStatus((BridgeNetworkStatus) descriptor);
+    } else if (descriptor instanceof ServerDescriptor && !relay) {
+      this.processBridgeServerDescriptor((ServerDescriptor) descriptor);
+    } else if (descriptor instanceof BridgePoolAssignment) {
+      this.processBridgePoolAssignment((BridgePoolAssignment) descriptor);
+    }
+  }
+
+  public void updateStatuses() {
+    this.readStatusSummary();
+    Logger.printStatusTime("Read status summary");
+    this.setCurrentNodes();
+    Logger.printStatusTime("Set current node fingerprints");
+    this.startReverseDomainNameLookups();
+    Logger.printStatusTime("Started reverse domain name lookups");
+    this.lookUpCitiesAndASes();
+    Logger.printStatusTime("Looked up cities and ASes");
+    this.setRunningBits();
+    Logger.printStatusTime("Set running bits");
+    this.calculatePathSelectionProbabilities();
+    Logger.printStatusTime("Calculated path selection probabilities");
+    this.finishReverseDomainNameLookups();
+    Logger.printStatusTime("Finished reverse domain name lookups");
+    this.writeStatusSummary();
+    Logger.printStatusTime("Wrote status summary");
+    /* TODO Does anything break if we take the following out?
+     * Like, does DocumentStore make sure there's a status/summary with
+     * all node statuses and an out/summary with only recent ones?
+    this.writeOutSummary();
+    Logger.printStatusTime("Wrote out summary");*/
+  }
+
+  private void processRelayNetworkStatusConsensus(
+      RelayNetworkStatusConsensus consensus) {
+    long validAfterMillis = consensus.getValidAfterMillis();
+    if (validAfterMillis > this.relaysLastValidAfterMillis) {
+      this.relaysLastValidAfterMillis = validAfterMillis;
+    }
+    Set<String> recommendedVersions = null;
+    if (consensus.getRecommendedServerVersions() != null) {
+      recommendedVersions = new HashSet<String>();
+      for (String recommendedVersion :
+          consensus.getRecommendedServerVersions()) {
+        recommendedVersions.add("Tor " + recommendedVersion);
+      }
+    }
+    for (NetworkStatusEntry entry :
+        consensus.getStatusEntries().values()) {
+      String nickname = entry.getNickname();
+      String fingerprint = entry.getFingerprint();
+      String address = entry.getAddress();
+      SortedSet<String> orAddressesAndPorts = new TreeSet<String>(
+          entry.getOrAddresses());
+      int orPort = entry.getOrPort();
+      int dirPort = entry.getDirPort();
+      SortedSet<String> relayFlags = entry.getFlags();
+      long consensusWeight = entry.getBandwidth();
+      String defaultPolicy = entry.getDefaultPolicy();
+      String portList = entry.getPortList();
+      Boolean recommendedVersion = (recommendedVersions == null ||
+          entry.getVersion() == null) ? null :
+          recommendedVersions.contains(entry.getVersion());
+      NodeStatus newNodeStatus = new NodeStatus(true, nickname,
+          fingerprint, address, orAddressesAndPorts, null,
+          validAfterMillis, orPort, dirPort, relayFlags, consensusWeight,
+          null, null, -1L, defaultPolicy, portList, validAfterMillis,
+          validAfterMillis, null, null, recommendedVersion);
+      if (this.knownNodes.containsKey(fingerprint)) {
+        this.knownNodes.get(fingerprint).update(newNodeStatus);
+      } else {
+        this.knownNodes.put(fingerprint, newNodeStatus);
+      }
+    }
+    this.relayConsensusesProcessed++;
+    if (this.relaysLastValidAfterMillis == validAfterMillis) {
+      this.lastBandwidthWeights = consensus.getBandwidthWeights();
+    }
+  }
+
+  private void processBridgeNetworkStatus(BridgeNetworkStatus status) {
+    long publishedMillis = status.getPublishedMillis();
+    if (publishedMillis > this.bridgesLastPublishedMillis) {
+      this.bridgesLastPublishedMillis = publishedMillis;
+    }
+    for (NetworkStatusEntry entry : status.getStatusEntries().values()) {
+      String nickname = entry.getNickname();
+      String fingerprint = entry.getFingerprint();
+      String address = entry.getAddress();
+      SortedSet<String> orAddressesAndPorts = new TreeSet<String>(
+          entry.getOrAddresses());
+      int orPort = entry.getOrPort();
+      int dirPort = entry.getDirPort();
+      SortedSet<String> relayFlags = entry.getFlags();
+      NodeStatus newNodeStatus = new NodeStatus(false, nickname,
+          fingerprint, address, orAddressesAndPorts, null,
+          publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null,
+          -1L, null, null, publishedMillis, -1L, null, null, null);
+      if (this.knownNodes.containsKey(fingerprint)) {
+        this.knownNodes.get(fingerprint).update(newNodeStatus);
+      } else {
+        this.knownNodes.put(fingerprint, newNodeStatus);
+      }
+    }
+    this.bridgeStatusesProcessed++;
+  }
+
+  private void readStatusSummary() {
+    SortedSet<String> fingerprints = this.documentStore.list(
+        NodeStatus.class, true);
+    for (String fingerprint : fingerprints) {
+      NodeStatus node = this.documentStore.retrieve(NodeStatus.class,
+          true, fingerprint);
+      if (node.isRelay()) {
+        this.relaysLastValidAfterMillis = Math.max(
+            this.relaysLastValidAfterMillis, node.getLastSeenMillis());
+      } else {
+        this.bridgesLastPublishedMillis = Math.max(
+            this.bridgesLastPublishedMillis, node.getLastSeenMillis());
+      }
+      if (this.knownNodes.containsKey(fingerprint)) {
+        this.knownNodes.get(fingerprint).update(node);
+      } else {
+        this.knownNodes.put(fingerprint, node);
+      }
+    }
+  }
+
+  private void setRunningBits() {
+    for (NodeStatus node : this.knownNodes.values()) {
+      if (node.isRelay() && node.getRelayFlags().contains("Running") &&
+          node.getLastSeenMillis() == this.relaysLastValidAfterMillis) {
+        node.setRunning(true);
+      }
+      if (!node.isRelay() && node.getRelayFlags().contains("Running") &&
+          node.getLastSeenMillis() == this.bridgesLastPublishedMillis) {
+        node.setRunning(true);
+      }
+    }
+  }
+
+  private void lookUpCitiesAndASes() {
+    SortedSet<String> addressStrings = new TreeSet<String>();
+    for (NodeStatus node : this.knownNodes.values()) {
+      if (node.isRelay()) {
+        addressStrings.add(node.getAddress());
+      }
+    }
+    if (addressStrings.isEmpty()) {
+      System.err.println("No relay IP addresses to resolve to cities or "
+          + "ASN.");
+      return;
+    }
+    SortedMap<String, LookupResult> lookupResults =
+        this.lookupService.lookup(addressStrings);
+    for (NodeStatus node : knownNodes.values()) {
+      if (!node.isRelay()) {
+        continue;
+      }
+      String addressString = node.getAddress();
+      if (lookupResults.containsKey(addressString)) {
+        LookupResult lookupResult = lookupResults.get(addressString);
+        node.setCountryCode(lookupResult.countryCode);
+        node.setCountryName(lookupResult.countryName);
+        node.setRegionName(lookupResult.regionName);
+        node.setCityName(lookupResult.cityName);
+        node.setLatitude(lookupResult.latitude);
+        node.setLongitude(lookupResult.longitude);
+        node.setASNumber(lookupResult.aSNumber);
+        node.setASName(lookupResult.aSName);
+      }
+    }
+  }
+
+  private void writeStatusSummary() {
+    this.writeSummary(true);
+  }
+
+  private void writeSummary(boolean includeArchive) {
+    SortedMap<String, NodeStatus> nodes = includeArchive
+        ? this.knownNodes : this.getCurrentNodes();
+    for (Map.Entry<String, NodeStatus> e : nodes.entrySet()) {
+      this.documentStore.store(e.getValue(), e.getKey());
+    }
+  }
+
+  private SortedMap<String, NodeStatus> getCurrentNodes() {
+    long cutoff = Math.max(this.relaysLastValidAfterMillis,
+        this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L;
+    SortedMap<String, NodeStatus> currentNodes =
+        new TreeMap<String, NodeStatus>();
+    for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) {
+      if (e.getValue().getLastSeenMillis() >= cutoff) {
+        currentNodes.put(e.getKey(), e.getValue());
+      }
+    }
+    return currentNodes;
+  }
+
+  private void processRelayServerDescriptor(
+      ServerDescriptor descriptor) {
+    String fingerprint = descriptor.getFingerprint();
+    DetailsStatus detailsStatus = this.documentStore.retrieve(
+        DetailsStatus.class, false, fingerprint);
+    SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setLenient(false);
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    String publishedDateTime =
+        dateTimeFormat.format(descriptor.getPublishedMillis());
+    if (detailsStatus != null) {
+      String detailsString = detailsStatus.documentString;
+      String descPublishedLine = "\"desc_published\":\""
+          + publishedDateTime + "\",";
+      Scanner s = new Scanner(detailsString);
+      while (s.hasNextLine()) {
+        String line = s.nextLine();
+        if (line.startsWith("\"desc_published\":\"")) {
+          if (descPublishedLine.compareTo(line) < 0) {
+            return;
+          } else {
+            break;
+          }
+        }
+      }
+      s.close();
+    }
+    StringBuilder sb = new StringBuilder();
+    String lastRestartedString = dateTimeFormat.format(
+        descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L);
+    int bandwidthRate = descriptor.getBandwidthRate();
+    int bandwidthBurst = descriptor.getBandwidthBurst();
+    int observedBandwidth = descriptor.getBandwidthObserved();
+    int advertisedBandwidth = Math.min(bandwidthRate,
+        Math.min(bandwidthBurst, observedBandwidth));
+    sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n"
+        + "\"last_restarted\":\"" + lastRestartedString + "\",\n"
+        + "\"bandwidth_rate\":" + bandwidthRate + ",\n"
+        + "\"bandwidth_burst\":" + bandwidthBurst + ",\n"
+        + "\"observed_bandwidth\":" + observedBandwidth + ",\n"
+        + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n"
+        + "\"exit_policy\":[");
+    int written = 0;
+    for (String exitPolicyLine : descriptor.getExitPolicyLines()) {
+      sb.append((written++ > 0 ? "," : "") + "\n  \"" + exitPolicyLine
+          + "\"");
+    }
+    sb.append("\n]");
+    if (descriptor.getContact() != null) {
+      sb.append(",\n\"contact\":\""
+          + escapeJSON(descriptor.getContact()) + "\"");
+    }
+    if (descriptor.getPlatform() != null) {
+      sb.append(",\n\"platform\":\""
+          + escapeJSON(descriptor.getPlatform()) + "\"");
+    }
+    if (descriptor.getFamilyEntries() != null) {
+      sb.append(",\n\"family\":[");
+      written = 0;
+      for (String familyEntry : descriptor.getFamilyEntries()) {
+        sb.append((written++ > 0 ? "," : "") + "\n  \"" + familyEntry
+            + "\"");
+      }
+      sb.append("\n]");
+    }
+    if (descriptor.getIpv6DefaultPolicy() != null &&
+        (descriptor.getIpv6DefaultPolicy().equals("accept") ||
+        descriptor.getIpv6DefaultPolicy().equals("reject")) &&
+        descriptor.getIpv6PortList() != null) {
+      sb.append(",\n\"exit_policy_v6_summary\":{\""
+          + descriptor.getIpv6DefaultPolicy() + "\":[");
+      int portsWritten = 0;
+      for (String portOrPortRange :
+          descriptor.getIpv6PortList().split(",")) {
+        sb.append((portsWritten++ > 0 ? "," : "") + "\"" + portOrPortRange
+            + "\"");
+      }
+      sb.append("]}");
+    }
+    if (descriptor.isHibernating()) {
+      sb.append(",\n\"hibernating\":true");
+    }
+    detailsStatus = new DetailsStatus();
+    detailsStatus.documentString = sb.toString();
+    this.documentStore.store(detailsStatus, fingerprint);
+  }
+
+  private void processBridgeServerDescriptor(
+      ServerDescriptor descriptor) {
+    String fingerprint = descriptor.getFingerprint();
+    DetailsStatus detailsStatus = this.documentStore.retrieve(
+        DetailsStatus.class, false, fingerprint);
+    SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setLenient(false);
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    String publishedDateTime =
+        dateTimeFormat.format(descriptor.getPublishedMillis());
+    String poolAssignmentLine = null;
+    if (detailsStatus != null) {
+      String detailsString = detailsStatus.documentString;
+      String descPublishedLine = "\"desc_published\":\""
+          + publishedDateTime + "\",";
+      Scanner s = new Scanner(detailsString);
+      while (s.hasNextLine()) {
+        String line = s.nextLine();
+        if (line.startsWith("\"pool_assignment\":")) {
+          poolAssignmentLine = line;
+        } else if (line.startsWith("\"desc_published\":") &&
+            descPublishedLine.compareTo(line) < 0) {
+          return;
+        }
+      }
+      s.close();
+    }
+    StringBuilder sb = new StringBuilder();
+    String lastRestartedString = dateTimeFormat.format(
+        descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L);
+    int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(),
+        Math.min(descriptor.getBandwidthBurst(),
+        descriptor.getBandwidthObserved()));
+    sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n"
+        + "\"last_restarted\":\"" + lastRestartedString + "\",\n"
+        + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n"
+        + "\"platform\":\"" + escapeJSON(descriptor.getPlatform())
+        + "\"");
+    if (poolAssignmentLine != null) {
+      sb.append(",\n" + poolAssignmentLine);
+    }
+    detailsStatus = new DetailsStatus();
+    detailsStatus.documentString = sb.toString();
+    this.documentStore.store(detailsStatus, fingerprint);
+  }
+
+  private static String escapeJSON(String s) {
+    return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'");
+  }
+
+  private void processBridgePoolAssignment(
+      BridgePoolAssignment bridgePoolAssignment) {
+    for (Map.Entry<String, String> e :
+        bridgePoolAssignment.getEntries().entrySet()) {
+      String fingerprint = e.getKey();
+      String details = e.getValue();
+      StringBuilder sb = new StringBuilder();
+      DetailsStatus detailsStatus = this.documentStore.retrieve(
+          DetailsStatus.class, false, fingerprint);
+      if (detailsStatus != null) {
+        String detailsString = detailsStatus.documentString;
+        Scanner s = new Scanner(detailsString);
+        int linesWritten = 0;
+        boolean endsWithComma = false;
+        while (s.hasNextLine()) {
+          String line = s.nextLine();
+          if (!line.startsWith("\"pool_assignment\":")) {
+            sb.append((linesWritten++ > 0 ? "\n" : "") + line);
+            endsWithComma = line.endsWith(",");
+          }
+        }
+        s.close();
+        if (sb.length() > 0) {
+          sb.append((endsWithComma ? "" : ",") + "\n");
+        }
+      }
+      sb.append("\"pool_assignment\":\"" + details + "\"");
+      detailsStatus = new DetailsStatus();
+      detailsStatus.documentString = sb.toString();
+      this.documentStore.store(detailsStatus, fingerprint);
+    }
+  }
+
+  private void setCurrentNodes() {
+    SortedMap<String, NodeStatus> currentNodes = this.getCurrentNodes();
+    this.relays = new TreeMap<String, NodeStatus>();
+    this.bridges = new TreeMap<String, NodeStatus>();
+    for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) {
+      if (e.getValue().isRelay()) {
+        this.relays.put(e.getKey(), e.getValue());
+      } else {
+        this.bridges.put(e.getKey(), e.getValue());
+      }
+    }
+  }
+
+  private void startReverseDomainNameLookups() {
+    Map<String, Long> addressLastLookupTimes =
+        new HashMap<String, Long>();
+    for (NodeStatus relay : relays.values()) {
+      addressLastLookupTimes.put(relay.getAddress(),
+          relay.getLastRdnsLookup());
+    }
+    this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes);
+    this.reverseDomainNameResolver.startReverseDomainNameLookups();
+  }
+
+  private void finishReverseDomainNameLookups() {
+    this.reverseDomainNameResolver.finishReverseDomainNameLookups();
+    Map<String, String> lookupResults =
+        this.reverseDomainNameResolver.getLookupResults();
+    long startedRdnsLookups =
+        this.reverseDomainNameResolver.getLookupStartMillis();
+    for (NodeStatus relay : relays.values()) {
+      if (lookupResults.containsKey(relay.getAddress())) {
+        relay.setHostName(lookupResults.get(relay.getAddress()));
+        relay.setLastRdnsLookup(startedRdnsLookups);
+      }
+    }
+  }
+
+  private void calculatePathSelectionProbabilities() {
+    boolean consensusContainsBandwidthWeights = false;
+    double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0,
+        wmd = 0.0, wee = 0.0, wed = 0.0;
+    if (this.lastBandwidthWeights != null) {
+      SortedSet<String> weightKeys = new TreeSet<String>(Arrays.asList(
+          "Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(",")));
+      weightKeys.removeAll(this.lastBandwidthWeights.keySet());
+      if (weightKeys.isEmpty()) {
+        consensusContainsBandwidthWeights = true;
+        wgg = ((double) this.lastBandwidthWeights.get("Wgg")) / 10000.0;
+        wgd = ((double) this.lastBandwidthWeights.get("Wgd")) / 10000.0;
+        wmg = ((double) this.lastBandwidthWeights.get("Wmg")) / 10000.0;
+        wmm = ((double) this.lastBandwidthWeights.get("Wmm")) / 10000.0;
+        wme = ((double) this.lastBandwidthWeights.get("Wme")) / 10000.0;
+        wmd = ((double) this.lastBandwidthWeights.get("Wmd")) / 10000.0;
+        wee = ((double) this.lastBandwidthWeights.get("Wee")) / 10000.0;
+        wed = ((double) this.lastBandwidthWeights.get("Wed")) / 10000.0;
+      }
+    } else {
+      System.err.println("Could not determine most recent Wxx parameter "
+          + "values, probably because we didn't parse a consensus in "
+          + "this execution.  All relays' guard/middle/exit weights are "
+          + "going to be 0.0.");
+    }
+    SortedMap<String, Double>
+        advertisedBandwidths = new TreeMap<String, Double>(),
+        consensusWeights = new TreeMap<String, Double>(),
+        guardWeights = new TreeMap<String, Double>(),
+        middleWeights = new TreeMap<String, Double>(),
+        exitWeights = new TreeMap<String, Double>();
+    double totalAdvertisedBandwidth = 0.0;
+    double totalConsensusWeight = 0.0;
+    double totalGuardWeight = 0.0;
+    double totalMiddleWeight = 0.0;
+    double totalExitWeight = 0.0;
+    for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) {
+      String fingerprint = e.getKey();
+      NodeStatus relay = e.getValue();
+      if (!relay.getRunning()) {
+        continue;
+      }
+      boolean isExit = relay.getRelayFlags().contains("Exit") &&
+          !relay.getRelayFlags().contains("BadExit");
+      boolean isGuard = relay.getRelayFlags().contains("Guard");
+      DetailsStatus detailsStatus = this.documentStore.retrieve(
+          DetailsStatus.class, false, fingerprint);
+      if (detailsStatus != null) {
+        double advertisedBandwidth = -1.0;
+        String detailsString = detailsStatus.documentString;
+        Scanner s = new Scanner(detailsString);
+        while (s.hasNextLine()) {
+          String line = s.nextLine();
+          if (!line.startsWith("\"advertised_bandwidth\":")) {
+            continue;
+          }
+          try {
+            advertisedBandwidth = (double) Integer.parseInt(
+                line.split(":")[1].replaceAll(",", ""));
+          } catch (NumberFormatException ex) {
+            /* Handle below. */
+          }
+          break;
+        }
+        s.close();
+        if (advertisedBandwidth >= 0.0) {
+          advertisedBandwidths.put(fingerprint, advertisedBandwidth);
+          totalAdvertisedBandwidth += advertisedBandwidth;
+        }
+      }
+      double consensusWeight = (double) relay.getConsensusWeight();
+      consensusWeights.put(fingerprint, consensusWeight);
+      totalConsensusWeight += consensusWeight;
+      if (consensusContainsBandwidthWeights) {
+        double guardWeight = consensusWeight,
+            middleWeight = consensusWeight,
+            exitWeight = consensusWeight;
+        if (isGuard && isExit) {
+          guardWeight *= wgd;
+          middleWeight *= wmd;
+          exitWeight *= wed;
+        } else if (isGuard) {
+          guardWeight *= wgg;
+          middleWeight *= wmg;
+          exitWeight = 0.0;
+        } else if (isExit) {
+          guardWeight = 0.0;
+          middleWeight *= wme;
+          exitWeight *= wee;
+        } else {
+          guardWeight = 0.0;
+          middleWeight *= wmm;
+          exitWeight = 0.0;
+        }
+        guardWeights.put(fingerprint, guardWeight);
+        middleWeights.put(fingerprint, middleWeight);
+        exitWeights.put(fingerprint, exitWeight);
+        totalGuardWeight += guardWeight;
+        totalMiddleWeight += middleWeight;
+        totalExitWeight += exitWeight;
+      }
+    }
+    for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) {
+      String fingerprint = e.getKey();
+      NodeStatus relay = e.getValue();
+      if (advertisedBandwidths.containsKey(fingerprint)) {
+        relay.setAdvertisedBandwidthFraction(advertisedBandwidths.get(
+            fingerprint) / totalAdvertisedBandwidth);
+      }
+      if (consensusWeights.containsKey(fingerprint)) {
+        relay.setConsensusWeightFraction(consensusWeights.get(fingerprint)
+            / totalConsensusWeight);
+      }
+      if (guardWeights.containsKey(fingerprint)) {
+        relay.setGuardProbability(guardWeights.get(fingerprint)
+            / totalGuardWeight);
+      }
+      if (middleWeights.containsKey(fingerprint)) {
+        relay.setMiddleProbability(middleWeights.get(fingerprint)
+            / totalMiddleWeight);
+      }
+      if (exitWeights.containsKey(fingerprint)) {
+        relay.setExitProbability(exitWeights.get(fingerprint)
+            / totalExitWeight);
+      }
+    }
+  }
+
+  public String getStatsString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("    " + Logger.formatDecimalNumber(
+        relayConsensusesProcessed) + " relay consensuses processed\n");
+    sb.append("    " + Logger.formatDecimalNumber(bridgeStatusesProcessed)
+        + " bridge statuses processed\n");
+    return sb.toString();
+  }
+}
+

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits