[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [onionoo/master] Add last_changed_address_or_port field to details.



commit 513b48093fc104201f7f4b31f5a17543297e3c85
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Wed Mar 27 19:26:54 2013 +0100

    Add last_changed_address_or_port field to details.
    
    Implements Onionoo side of #8374.
---
 src/org/torproject/onionoo/CurrentNodes.java     |  168 +++++++++++++++-------
 src/org/torproject/onionoo/DetailDataWriter.java |    4 +
 src/org/torproject/onionoo/Node.java             |   31 ++++-
 web/index.html                                   |   14 ++
 4 files changed, 161 insertions(+), 56 deletions(-)

diff --git a/src/org/torproject/onionoo/CurrentNodes.java b/src/org/torproject/onionoo/CurrentNodes.java
index 487cf4d..d98eb4e 100644
--- a/src/org/torproject/onionoo/CurrentNodes.java
+++ b/src/org/torproject/onionoo/CurrentNodes.java
@@ -11,6 +11,7 @@ import java.io.IOException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -118,18 +119,25 @@ public class CurrentNodes {
             firstSeenMillis = dateTimeFormat.parse(parts[15] + " "
                 + parts[16]).getTime();
           }
+          long lastChangedAddresses = publishedOrValidAfterMillis;
+          if (parts.length > 18 && !parts[17].equals("null")) {
+            lastChangedAddresses = dateTimeFormat.parse(parts[17] + " "
+                + parts[18]).getTime();
+          }
           if (isRelay) {
             this.addRelay(nickname, fingerprint, address,
                 orAddressesAndPorts, exitAddresses,
                 publishedOrValidAfterMillis, orPort, dirPort, relayFlags,
                 consensusWeight, countryCode, hostName, lastRdnsLookup,
-                defaultPolicy, portList, firstSeenMillis);
+                defaultPolicy, portList, firstSeenMillis,
+                lastChangedAddresses);
           } else {
             this.addBridge(nickname, fingerprint, address,
                 orAddressesAndPorts, exitAddresses,
                 publishedOrValidAfterMillis, orPort, dirPort, relayFlags,
                 consensusWeight, countryCode, hostName, lastRdnsLookup,
-                defaultPolicy, portList, firstSeenMillis);
+                defaultPolicy, portList, firstSeenMillis,
+                lastChangedAddresses);
           }
         }
         br.close();
@@ -158,7 +166,13 @@ public class CurrentNodes {
       SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
           "yyyy-MM-dd HH:mm:ss");
       dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+      long cutoff = System.currentTimeMillis()
+          - 7L * 24L * 60L * 60L * 1000L;
       for (Node entry : this.currentRelays.values()) {
+        long lastSeenMillis = entry.getLastSeenMillis();
+        if (lastSeenMillis < cutoff) {
+          continue;
+        }
         String nickname = entry.getNickname();
         String fingerprint = entry.getFingerprint();
         String address = entry.getAddress();
@@ -175,8 +189,7 @@ public class CurrentNodes {
           addressesBuilder.append((written++ > 0 ? "+" : "")
               + exitAddress);
         }
-        String validAfter = dateTimeFormat.format(
-            entry.getLastSeenMillis());
+        String lastSeen = dateTimeFormat.format(lastSeenMillis);
         String orPort = String.valueOf(entry.getOrPort());
         String dirPort = String.valueOf(entry.getDirPort());
         StringBuilder flagsBuilder = new StringBuilder();
@@ -197,12 +210,15 @@ public class CurrentNodes {
             ? entry.getPortList() : "null";
         String firstSeen = dateTimeFormat.format(
             entry.getFirstSeenMillis());
+        String lastChangedAddresses = dateTimeFormat.format(
+            entry.getLastChangedOrAddress());
         bw.write("r " + nickname + " " + fingerprint + " "
-            + addressesBuilder.toString() + " " + validAfter + " "
+            + addressesBuilder.toString() + " " + lastSeen + " "
             + orPort + " " + dirPort + " " + flagsBuilder.toString() + " "
             + consensusWeight + " " + countryCode + " " + hostName + " "
             + String.valueOf(lastRdnsLookup) + " " + defaultPolicy + " "
-            + portList + " " + firstSeen + "\n");
+            + portList + " " + firstSeen + " " + lastChangedAddresses
+            + "\n");
       }
       for (Node entry : this.currentBridges.values()) {
         String nickname = entry.getNickname();
@@ -230,7 +246,7 @@ public class CurrentNodes {
         bw.write("b " + nickname + " " + fingerprint + " "
             + addressesBuilder.toString() + " " + published + " " + orPort
             + " " + dirPort + " " + flagsBuilder.toString()
-            + " -1 ?? null -1 null null " + firstSeen + "\n");
+            + " -1 ?? null -1 null null " + firstSeen + " null null\n");
       }
       bw.close();
     } catch (IOException e) {
@@ -245,9 +261,6 @@ public class CurrentNodes {
   private long lastValidAfterMillis = 0L;
   private long lastPublishedMillis = 0L;
 
-  private long cutoff = System.currentTimeMillis()
-      - 7L * 24L * 60L * 60L * 1000L;
-
   public void readRelayNetworkConsensuses() {
     DescriptorReader reader =
         DescriptorSourceFactory.createDescriptorReader();
@@ -304,7 +317,7 @@ public class CurrentNodes {
       this.addRelay(nickname, fingerprint, address, orAddressesAndPorts,
           null, validAfterMillis, orPort, dirPort, relayFlags,
           consensusWeight, null, null, -1L, defaultPolicy, portList,
-          validAfterMillis);
+          validAfterMillis, validAfterMillis);
     }
     if (this.lastValidAfterMillis == validAfterMillis) {
       this.lastBandwidthWeights = consensus.getBandwidthWeights();
@@ -313,33 +326,62 @@ public class CurrentNodes {
 
   public void addRelay(String nickname, String fingerprint,
       String address, SortedSet<String> orAddressesAndPorts,
-      SortedSet<String> exitAddresses, long validAfterMillis, int orPort,
+      SortedSet<String> exitAddresses, long lastSeenMillis, int orPort,
       int dirPort, SortedSet<String> relayFlags, long consensusWeight,
       String countryCode, String hostName, long lastRdnsLookup,
-      String defaultPolicy, String portList, long firstSeenMillis) {
-    if (validAfterMillis >= cutoff &&
-        (!this.currentRelays.containsKey(fingerprint) ||
-        this.currentRelays.get(fingerprint).getLastSeenMillis() <
-        validAfterMillis)) {
-      Node previousRelay = this.currentRelays.containsKey(fingerprint)
-          ? this.currentRelays.get(fingerprint) : null;
-      if (previousRelay != null && hostName == null &&
-          previousRelay.getAddress().equals(address)) {
-        hostName = previousRelay.getHostName();
-        lastRdnsLookup = previousRelay.getLastRdnsLookup();
-      }
-      if (previousRelay != null) {
-        firstSeenMillis = Math.min(firstSeenMillis,
-            previousRelay.getFirstSeenMillis());
+      String defaultPolicy, String portList, long firstSeenMillis,
+      long lastChangedAddresses) {
+    /* Remember addresses and OR/dir ports that the relay advertised at
+     * the given time. */
+    SortedMap<Long, Set<String>> lastAddresses =
+        new TreeMap<Long, Set<String>>(Collections.reverseOrder());
+    Set<String> addresses = new HashSet<String>();
+    addresses.add(address + ":" + orPort);
+    if (dirPort > 0) {
+      addresses.add(address + ":" + dirPort);
+    }
+    addresses.addAll(orAddressesAndPorts);
+    lastAddresses.put(lastChangedAddresses, addresses);
+    /* See if there's already an entry for this relay. */
+    if (this.currentRelays.containsKey(fingerprint)) {
+      Node existingEntry = this.currentRelays.get(fingerprint);
+      if (lastSeenMillis < existingEntry.getLastSeenMillis()) {
+        /* Use latest information for nickname, current addresses, etc. */
+        nickname = existingEntry.getNickname();
+        address = existingEntry.getAddress();
+        orAddressesAndPorts = existingEntry.getOrAddressesAndPorts();
+        exitAddresses = existingEntry.getExitAddresses();
+        lastSeenMillis = existingEntry.getLastSeenMillis();
+        orPort = existingEntry.getOrPort();
+        dirPort = existingEntry.getDirPort();
+        relayFlags = existingEntry.getRelayFlags();
+        consensusWeight = existingEntry.getConsensusWeight();
+        countryCode = existingEntry.getCountryCode();
+        defaultPolicy = existingEntry.getDefaultPolicy();
+        portList = existingEntry.getPortList();
       }
-      Node entry = new Node(nickname, fingerprint, address,
-          orAddressesAndPorts, exitAddresses, validAfterMillis, orPort,
-          dirPort, relayFlags, consensusWeight, countryCode, hostName,
-          lastRdnsLookup, defaultPolicy, portList, firstSeenMillis);
-      this.currentRelays.put(fingerprint, entry);
-      if (validAfterMillis > this.lastValidAfterMillis) {
-        this.lastValidAfterMillis = validAfterMillis;
+      if (hostName == null &&
+          existingEntry.getAddress().equals(address)) {
+        /* Re-use reverse DNS lookup results if available. */
+        hostName = existingEntry.getHostName();
+        lastRdnsLookup = existingEntry.getLastRdnsLookup();
       }
+      /* Update relay-history fields. */
+      firstSeenMillis = Math.min(firstSeenMillis,
+          existingEntry.getFirstSeenMillis());
+      lastAddresses.putAll(existingEntry.getLastAddresses());
+    }
+    /* Add or update entry. */
+    Node entry = new Node(nickname, fingerprint, address,
+        orAddressesAndPorts, exitAddresses, lastSeenMillis, orPort,
+        dirPort, relayFlags, consensusWeight, countryCode, hostName,
+        lastRdnsLookup, defaultPolicy, portList, firstSeenMillis,
+        lastAddresses);
+    this.currentRelays.put(fingerprint, entry);
+    /* If this entry comes from a new consensus, update our global last
+     * valid-after time. */
+    if (lastSeenMillis > this.lastValidAfterMillis) {
+      this.lastValidAfterMillis = lastSeenMillis;
     }
   }
 
@@ -673,7 +715,8 @@ public class CurrentNodes {
           }
         }
         if (addressNumberASN.containsKey(addressNumber)) {
-          String[] parts = addressNumberASN.get(addressNumber).split(" ", 2);
+          String[] parts = addressNumberASN.get(addressNumber).split(" ",
+              2);
           relay.setASNumber(parts[0]);
           relay.setASName(parts[1]);
         }
@@ -726,34 +769,49 @@ public class CurrentNodes {
       SortedSet<String> relayFlags = entry.getFlags();
       this.addBridge(nickname, fingerprint, address, orAddressesAndPorts,
           null, publishedMillis, orPort, dirPort, relayFlags, -1, "??",
-          null, -1L, null, null, publishedMillis);
+          null, -1L, null, null, publishedMillis, -1L);
     }
   }
 
   public void addBridge(String nickname, String fingerprint,
       String address, SortedSet<String> orAddressesAndPorts,
-      SortedSet<String> exitAddresses, long publishedMillis, int orPort,
+      SortedSet<String> exitAddresses, long lastSeenMillis, int orPort,
       int dirPort, SortedSet<String> relayFlags, long consensusWeight,
       String countryCode, String hostname, long lastRdnsLookup,
-      String defaultPolicy, String portList, long firstSeenMillis) {
-    if (publishedMillis >= cutoff &&
-        (!this.currentBridges.containsKey(fingerprint) ||
-        this.currentBridges.get(fingerprint).getLastSeenMillis() <
-        publishedMillis)) {
-      Node previousBridge = this.currentBridges.containsKey(fingerprint)
-          ? this.currentBridges.get(fingerprint) : null;
-      if (previousBridge != null) {
-        firstSeenMillis = Math.min(firstSeenMillis,
-            previousBridge.getFirstSeenMillis());
-      }
-      Node entry = new Node(nickname, fingerprint, address,
-          orAddressesAndPorts, exitAddresses, publishedMillis, orPort,
-          dirPort, relayFlags, consensusWeight, countryCode, hostname,
-          lastRdnsLookup, defaultPolicy, portList, firstSeenMillis);
-      this.currentBridges.put(fingerprint, entry);
-      if (publishedMillis > this.lastPublishedMillis) {
-        this.lastPublishedMillis = publishedMillis;
+      String defaultPolicy, String portList, long firstSeenMillis,
+      long lastChangedAddresses) {
+    /* See if there's already an entry for this bridge. */
+    if (this.currentBridges.containsKey(fingerprint)) {
+      Node existingEntry = this.currentBridges.get(fingerprint);
+      if (lastSeenMillis < existingEntry.getLastSeenMillis()) {
+        /* Use latest information for nickname, current addresses, etc. */
+        nickname = existingEntry.getNickname();
+        address = existingEntry.getAddress();
+        orAddressesAndPorts = existingEntry.getOrAddressesAndPorts();
+        exitAddresses = existingEntry.getExitAddresses();
+        lastSeenMillis = existingEntry.getLastSeenMillis();
+        orPort = existingEntry.getOrPort();
+        dirPort = existingEntry.getDirPort();
+        relayFlags = existingEntry.getRelayFlags();
+        consensusWeight = existingEntry.getConsensusWeight();
+        countryCode = existingEntry.getCountryCode();
+        defaultPolicy = existingEntry.getDefaultPolicy();
+        portList = existingEntry.getPortList();
       }
+      /* Update relay-history fields. */
+      firstSeenMillis = Math.min(firstSeenMillis,
+          existingEntry.getFirstSeenMillis());
+    }
+    /* Add or update entry. */
+    Node entry = new Node(nickname, fingerprint, address,
+        orAddressesAndPorts, exitAddresses, lastSeenMillis, orPort,
+        dirPort, relayFlags, consensusWeight, countryCode, hostname,
+        lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, null);
+    this.currentBridges.put(fingerprint, entry);
+    /* If this entry comes from a new status, update our global last
+     * published time. */
+    if (lastSeenMillis > this.lastPublishedMillis) {
+      this.lastPublishedMillis = lastSeenMillis;
     }
   }
 
diff --git a/src/org/torproject/onionoo/DetailDataWriter.java b/src/org/torproject/onionoo/DetailDataWriter.java
index dcea869..8236d8b 100644
--- a/src/org/torproject/onionoo/DetailDataWriter.java
+++ b/src/org/torproject/onionoo/DetailDataWriter.java
@@ -575,6 +575,8 @@ public class DetailDataWriter {
       String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis());
       String firstSeen = dateTimeFormat.format(
           entry.getFirstSeenMillis());
+      String lastChangedOrAddress = dateTimeFormat.format(
+          entry.getLastChangedOrAddress());
       String running = entry.getRunning() ? "true" : "false";
       int dirPort = entry.getDirPort();
       String countryCode = entry.getCountryCode();
@@ -607,6 +609,8 @@ public class DetailDataWriter {
       }
       sb.append(",\n\"last_seen\":\"" + lastSeen + "\"");
       sb.append(",\n\"first_seen\":\"" + firstSeen + "\"");
+      sb.append(",\n\"last_changed_address_or_port\":\""
+          + lastChangedOrAddress + "\"");
       sb.append(",\n\"running\":" + running);
       SortedSet<String> relayFlags = entry.getRelayFlags();
       if (!relayFlags.isEmpty()) {
diff --git a/src/org/torproject/onionoo/Node.java b/src/org/torproject/onionoo/Node.java
index 0857b4c..0cadd38 100644
--- a/src/org/torproject/onionoo/Node.java
+++ b/src/org/torproject/onionoo/Node.java
@@ -2,8 +2,12 @@
  * See LICENSE for licensing information */
 package org.torproject.onionoo;
 
+import java.util.Map;
+import java.util.Set;
 import java.util.SortedSet;
+import java.util.SortedMap;
 import java.util.TreeSet;
+import java.util.TreeMap;
 
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
@@ -43,12 +47,14 @@ public class Node {
   private double exitProbability = -1.0;
   private String defaultPolicy;
   private String portList;
+  private SortedMap<Long, Set<String>> lastAddresses;
   public Node(String nickname, String fingerprint, String address,
       SortedSet<String> orAddressesAndPorts,
       SortedSet<String> exitAddresses, long lastSeenMillis, int orPort,
       int dirPort, SortedSet<String> relayFlags, long consensusWeight,
       String countryCode, String hostName, long lastRdnsLookup,
-      String defaultPolicy, String portList, long firstSeenMillis) {
+      String defaultPolicy, String portList, long firstSeenMillis,
+      SortedMap<Long, Set<String>> lastAddresses) {
     this.nickname = nickname;
     this.fingerprint = fingerprint;
     try {
@@ -82,6 +88,7 @@ public class Node {
     this.defaultPolicy = defaultPolicy;
     this.portList = portList;
     this.firstSeenMillis = firstSeenMillis;
+    this.lastAddresses = lastAddresses;
   }
   public String getFingerprint() {
     return this.fingerprint;
@@ -246,5 +253,27 @@ public class Node {
   public String getPortList() {
     return this.portList;
   }
+  public SortedMap<Long, Set<String>> getLastAddresses() {
+    return this.lastAddresses == null ? null :
+        new TreeMap<Long, Set<String>>(this.lastAddresses);
+  }
+  public long getLastChangedOrAddress() {
+    long lastChangedAddressesMillis = -1L;
+    if (this.lastAddresses != null) {
+      Set<String> lastAddresses = null;
+      for (Map.Entry<Long, Set<String>> e : this.lastAddresses.entrySet()) {
+        if (lastAddresses != null) {
+          for (String address : e.getValue()) {
+            if (!lastAddresses.contains(address)) {
+              return lastChangedAddressesMillis;
+            }
+          }
+        }
+        lastChangedAddressesMillis = e.getKey();
+        lastAddresses = e.getValue();
+      }
+    }
+    return lastChangedAddressesMillis;
+  }
 }
 
diff --git a/web/index.html b/web/index.html
index 4c3491c..e3bbdb7 100755
--- a/web/index.html
+++ b/web/index.html
@@ -130,6 +130,20 @@ Omitted if the relay does not accept directory connections.</li>
 <li><b>"last_seen":</b> UTC timestamp (YYYY-MM-DD hh:mm:ss) when this
 relay was last seen in a network status consensus.
 Required field.</li>
+<li><font color="blue"><b>"last_changed_address_or_port":</b></font>
+UTC timestamp (YYYY-MM-DD
+hh:mm:ss) when this relay last stopped announcing an IPv4 or IPv6 address
+or TCP port where it previously accepted onion-routing or directory
+connections.
+This timestamp can serve as indicator whether this relay would be a
+suitable fallback directory.
+Note that this timestamp is reset when a relay drops out of the consensus
+for more than 7 days, so that whenever that relay rejoins, it would seem
+as if it changed its address(es) at that time; this is a known limitation
+of the current Onionoo design.
+Required field.
+<font color="blue">Added on March 27.</font>
+</li>
 <li><b>"first_seen":</b> UTC timestamp (YYYY-MM-DD hh:mm:ss) when this
 relay was first seen in a network status consensus.
 Note that this timestamp is reset when a relay drops out of the consensus

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits