[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [onionoo/master] Remember nodes even if they drop out for 7+ days.



commit 49595204ad73da7fca1a5285e693400195d66247
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Tue Apr 9 11:08:00 2013 +0200

    Remember nodes even if they drop out for 7+ days.
    
    Implements step 2 of #6509.
---
 src/org/torproject/onionoo/CurrentNodes.java |   93 ++++++++++++++-----------
 src/org/torproject/onionoo/Main.java         |    5 +-
 web/index.html                               |   27 ++++----
 3 files changed, 68 insertions(+), 57 deletions(-)

diff --git a/src/org/torproject/onionoo/CurrentNodes.java b/src/org/torproject/onionoo/CurrentNodes.java
index f2bbac7..db4305d 100644
--- a/src/org/torproject/onionoo/CurrentNodes.java
+++ b/src/org/torproject/onionoo/CurrentNodes.java
@@ -11,6 +11,7 @@ import java.io.IOException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -36,16 +37,12 @@ import org.torproject.descriptor.RelayNetworkStatusConsensus;
  * days. */
 public class CurrentNodes {
 
-  private File internalRelaySearchDataFile;
-
   /* Read the internal relay search data file from disk. */
-  public void readRelaySearchDataFile(File internalRelaySearchDataFile) {
-    this.internalRelaySearchDataFile = internalRelaySearchDataFile;
-    if (this.internalRelaySearchDataFile.exists() &&
-        !this.internalRelaySearchDataFile.isDirectory()) {
+  public void readRelaySearchDataFile(File summaryFile) {
+    if (summaryFile.exists() && !summaryFile.isDirectory()) {
       try {
         BufferedReader br = new BufferedReader(new FileReader(
-            this.internalRelaySearchDataFile));
+            summaryFile));
         String line;
         SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
             "yyyy-MM-dd HH:mm:ss");
@@ -55,7 +52,7 @@ public class CurrentNodes {
           boolean isRelay = parts[0].equals("r");
           if (parts.length < 9) {
             System.err.println("Line '" + line + "' in '"
-                + this.internalRelaySearchDataFile.getAbsolutePath()
+                + summaryFile.getAbsolutePath()
                 + "' is invalid.  Exiting.");
             System.exit(1);
           }
@@ -69,7 +66,7 @@ public class CurrentNodes {
             String[] addressParts = addresses.split(";", -1);
             if (addressParts.length != 3) {
               System.err.println("Line '" + line + "' in '"
-                  + this.internalRelaySearchDataFile.getAbsolutePath()
+                  + summaryFile.getAbsolutePath()
                   + " is invalid.  Exiting.");
               System.exit(1);
             }
@@ -147,14 +144,12 @@ public class CurrentNodes {
         br.close();
       } catch (IOException e) {
         System.err.println("Could not read "
-            + this.internalRelaySearchDataFile.getAbsolutePath()
-            + ".  Exiting.");
+            + summaryFile.getAbsolutePath() + ".  Exiting.");
         e.printStackTrace();
         System.exit(1);
       } catch (ParseException e) {
         System.err.println("Could not read "
-            + this.internalRelaySearchDataFile.getAbsolutePath()
-            + ".  Exiting.");
+            + summaryFile.getAbsolutePath() + ".  Exiting.");
         e.printStackTrace();
         System.exit(1);
       }
@@ -162,21 +157,18 @@ public class CurrentNodes {
   }
 
   /* Write the internal relay search data file to disk. */
-  public void writeRelaySearchDataFile() {
+  public void writeRelaySearchDataFile(File summaryFile,
+      boolean includeOldNodes) {
     try {
-      this.internalRelaySearchDataFile.getParentFile().mkdirs();
+      summaryFile.getParentFile().mkdirs();
       BufferedWriter bw = new BufferedWriter(new FileWriter(
-          this.internalRelaySearchDataFile));
+          summaryFile));
       SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
           "yyyy-MM-dd HH:mm:ss");
       dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-      long cutoff = System.currentTimeMillis()
-          - 7L * 24L * 60L * 60L * 1000L;
-      for (Node entry : this.currentRelays.values()) {
-        long lastSeenMillis = entry.getLastSeenMillis();
-        if (lastSeenMillis < cutoff) {
-          continue;
-        }
+      Collection<Node> relays = includeOldNodes
+          ? this.knownRelays.values() : this.getCurrentRelays().values();
+      for (Node entry : relays) {
         String nickname = entry.getNickname();
         String fingerprint = entry.getFingerprint();
         String address = entry.getAddress();
@@ -193,7 +185,8 @@ public class CurrentNodes {
           addressesBuilder.append((written++ > 0 ? "+" : "")
               + exitAddress);
         }
-        String lastSeen = dateTimeFormat.format(lastSeenMillis);
+        String lastSeen = dateTimeFormat.format(
+            entry.getLastSeenMillis());
         String orPort = String.valueOf(entry.getOrPort());
         String dirPort = String.valueOf(entry.getDirPort());
         StringBuilder flagsBuilder = new StringBuilder();
@@ -226,7 +219,10 @@ public class CurrentNodes {
             + portList + " " + firstSeen + " " + lastChangedAddresses
             + " " + aSNumber + "\n");
       }
-      for (Node entry : this.currentBridges.values()) {
+      Collection<Node> bridges = includeOldNodes
+          ? this.knownBridges.values()
+          : this.getCurrentBridges().values();
+      for (Node entry : bridges) {
         String nickname = entry.getNickname();
         String fingerprint = entry.getFingerprint();
         String published = dateTimeFormat.format(
@@ -258,8 +254,7 @@ public class CurrentNodes {
       bw.close();
     } catch (IOException e) {
       System.err.println("Could not write '"
-          + this.internalRelaySearchDataFile.getAbsolutePath()
-          + "' to disk.  Exiting.");
+          + summaryFile.getAbsolutePath() + "' to disk.  Exiting.");
       e.printStackTrace();
       System.exit(1);
     }
@@ -294,7 +289,7 @@ public class CurrentNodes {
 
   public void setRelayRunningBits() {
     if (this.lastValidAfterMillis > 0L) {
-      for (Node entry : this.currentRelays.values()) {
+      for (Node entry : this.knownRelays.values()) {
         entry.setRunning(entry.getLastSeenMillis() ==
             this.lastValidAfterMillis);
       }
@@ -350,8 +345,8 @@ public class CurrentNodes {
     addresses.addAll(orAddressesAndPorts);
     lastAddresses.put(lastChangedAddresses, addresses);
     /* See if there's already an entry for this relay. */
-    if (this.currentRelays.containsKey(fingerprint)) {
-      Node existingEntry = this.currentRelays.get(fingerprint);
+    if (this.knownRelays.containsKey(fingerprint)) {
+      Node existingEntry = this.knownRelays.get(fingerprint);
       if (lastSeenMillis < existingEntry.getLastSeenMillis()) {
         /* Use latest information for nickname, current addresses, etc. */
         nickname = existingEntry.getNickname();
@@ -384,7 +379,7 @@ public class CurrentNodes {
         dirPort, relayFlags, consensusWeight, countryCode, hostName,
         lastRdnsLookup, defaultPolicy, portList, firstSeenMillis,
         lastAddresses, aSNumber);
-    this.currentRelays.put(fingerprint, entry);
+    this.knownRelays.put(fingerprint, entry);
     /* If this entry comes from a new consensus, update our global last
      * valid-after time. */
     if (lastSeenMillis > this.lastValidAfterMillis) {
@@ -436,7 +431,7 @@ public class CurrentNodes {
     /* Obtain a map from relay IP address strings to numbers. */
     Map<String, Long> addressStringNumbers = new HashMap<String, Long>();
     Pattern ipv4Pattern = Pattern.compile("^[0-9\\.]{7,15}$");
-    for (Node relay : this.currentRelays.values()) {
+    for (Node relay : this.knownRelays.values()) {
       String addressString = relay.getAddress();
       long addressNumber = -1L;
       if (ipv4Pattern.matcher(addressString).matches()) {
@@ -695,7 +690,7 @@ public class CurrentNodes {
     }
 
     /* Finally, set relays' city and ASN information. */
-    for (Node relay : currentRelays.values()) {
+    for (Node relay : knownRelays.values()) {
       String addressString = relay.getAddress();
       if (addressStringNumbers.containsKey(addressString)) {
         long addressNumber = addressStringNumbers.get(addressString);
@@ -756,7 +751,7 @@ public class CurrentNodes {
 
   public void setBridgeRunningBits() {
     if (this.lastPublishedMillis > 0L) {
-      for (Node entry : this.currentBridges.values()) {
+      for (Node entry : this.knownBridges.values()) {
         entry.setRunning(entry.getRelayFlags().contains("Running") &&
             entry.getLastSeenMillis() == this.lastPublishedMillis);
       }
@@ -788,8 +783,8 @@ public class CurrentNodes {
       String defaultPolicy, String portList, long firstSeenMillis,
       long lastChangedAddresses, String aSNumber) {
     /* See if there's already an entry for this bridge. */
-    if (this.currentBridges.containsKey(fingerprint)) {
-      Node existingEntry = this.currentBridges.get(fingerprint);
+    if (this.knownBridges.containsKey(fingerprint)) {
+      Node existingEntry = this.knownBridges.get(fingerprint);
       if (lastSeenMillis < existingEntry.getLastSeenMillis()) {
         /* Use latest information for nickname, current addresses, etc. */
         nickname = existingEntry.getNickname();
@@ -816,7 +811,7 @@ public class CurrentNodes {
         dirPort, relayFlags, consensusWeight, countryCode, hostname,
         lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, null,
         aSNumber);
-    this.currentBridges.put(fingerprint, entry);
+    this.knownBridges.put(fingerprint, entry);
     /* If this entry comes from a new status, update our global last
      * published time. */
     if (lastSeenMillis > this.lastPublishedMillis) {
@@ -824,16 +819,32 @@ public class CurrentNodes {
     }
   }
 
-  private SortedMap<String, Node> currentRelays =
+  private SortedMap<String, Node> knownRelays =
       new TreeMap<String, Node>();
   public SortedMap<String, Node> getCurrentRelays() {
-    return new TreeMap<String, Node>(this.currentRelays);
+    long cutoff = System.currentTimeMillis()
+        - 7L * 24L * 60L * 60L * 1000L;
+    SortedMap<String, Node> currentRelays = new TreeMap<String, Node>();
+    for (Map.Entry<String, Node> e : this.knownRelays.entrySet()) {
+      if (e.getValue().getLastSeenMillis() >= cutoff) {
+        currentRelays.put(e.getKey(), e.getValue());
+      }
+    }
+    return currentRelays;
   }
 
-  private SortedMap<String, Node> currentBridges =
+  private SortedMap<String, Node> knownBridges =
       new TreeMap<String, Node>();
   public SortedMap<String, Node> getCurrentBridges() {
-    return new TreeMap<String, Node>(this.currentBridges);
+    long cutoff = System.currentTimeMillis()
+        - 7L * 24L * 60L * 60L * 1000L;
+    SortedMap<String, Node> currentBridges = new TreeMap<String, Node>();
+    for (Map.Entry<String, Node> e : this.knownBridges.entrySet()) {
+      if (e.getValue().getLastSeenMillis() >= cutoff) {
+        currentBridges.put(e.getKey(), e.getValue());
+      }
+    }
+    return currentBridges;
   }
 
   public long getLastValidAfterMillis() {
diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java
index e3e7c5b..af45221 100644
--- a/src/org/torproject/onionoo/Main.java
+++ b/src/org/torproject/onionoo/Main.java
@@ -11,12 +11,13 @@ public class Main {
 
     printStatus("Updating internal node list.");
     CurrentNodes cn = new CurrentNodes();
-    cn.readRelaySearchDataFile(new File("out/summary"));
+    cn.readRelaySearchDataFile(new File("status/summary"));
     cn.readRelayNetworkConsensuses();
     cn.setRelayRunningBits();
     cn.lookUpCitiesAndASes();
     cn.readBridgeNetworkStatuses();
     cn.setBridgeRunningBits();
+    cn.writeRelaySearchDataFile(new File("status/summary"), true);
 
     printStatus("Updating detail data.");
     DetailDataWriter ddw = new DetailDataWriter();
@@ -47,7 +48,7 @@ public class Main {
     wdw.deleteObsoleteWeightsDataFiles();
 
     printStatus("Updating summary data.");
-    cn.writeRelaySearchDataFile();
+    cn.writeRelaySearchDataFile(new File("out/summary"), false);
 
     printStatus("Terminating.");
   }
diff --git a/web/index.html b/web/index.html
index 152cec6..cfd857d 100755
--- a/web/index.html
+++ b/web/index.html
@@ -137,20 +137,19 @@ or TCP port where it previously accepted onion-routing or directory
 connections.
 This timestamp can serve as indicator whether this relay would be a
 suitable fallback directory.
-Note that this timestamp is reset when a relay drops out of the consensus
-for more than 7 days, so that whenever that relay rejoins, it would seem
-as if it changed its address(es) at that time; this is a known limitation
-of the current Onionoo design.
 Required field.
 <font color="blue">Added on March 27.</font>
+<font color="blue">On April 9, fixed limitation that timestamp is reset
+when a relay drops out of the consensus for more than 7 days and
+rejoins.</font>
 </li>
 <li><b>"first_seen":</b> UTC timestamp (YYYY-MM-DD hh:mm:ss) when this
 relay was first seen in a network status consensus.
-Note that this timestamp is reset when a relay drops out of the consensus
-for more than 7 days, so that whenever that relay rejoins, it would seem
-as if it joined for the first time; this is a known limitation of the
-current Onionoo design.
-Required field.</li>
+Required field.
+<font color="blue">On April 9, fixed limitation that timestamp is reset
+when a relay drops out of the consensus for more than 7 days and
+rejoins.</font>
+</li>
 <li><b>"running":</b> Boolean field saying whether this relay was listed as
 running in the last relay network status consensus.
 Required field.</li>
@@ -340,11 +339,11 @@ was last seen in a bridge network status.
 Required field.</li>
 <li><b>"first_seen":</b> UTC timestamp (YYYY-MM-DD hh:mm:ss) when this
 bridge was first seen in a bridge network status.
-Note that this timestamp is reset when a bridge drops out of the network
-status for more than 7 days, so that whenever that bridge rejoins, it
-would seem as if it joined for the first time; this is a known limitation
-of the current Onionoo design.
-Required field.</li>
+Required field.
+<font color="blue">On April 9, fixed limitation that timestamp is reset
+when a bridge drops out of the network status for more than 7 days and
+rejoins.</font>
+</li>
 <li><b>"running":</b> Boolean field saying whether this bridge was listed
 as running in the last bridge network status.
 Required field.</li>

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits