[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-db/master] Recognize when data sources become stale.



commit f7b58361aa6df5fd2afe40c38f9dd111e2820f89
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Sat Oct 27 13:52:42 2012 -0400

    Recognize when data sources become stale.
    
    Implements #3850.
---
 .../db/bridgedescs/SanitizedBridgesWriter.java     |   58 +++++++++++
 .../BridgePoolAssignmentsProcessor.java            |   16 +++
 .../ernie/db/exitlists/ExitListDownloader.java     |  100 +++++++++++++++-----
 .../ernie/db/relaydescs/ArchiveWriter.java         |   46 +++++++++
 4 files changed, 194 insertions(+), 26 deletions(-)

diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
index 87593bd..7de9961 100644
--- a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
@@ -193,6 +193,8 @@ public class SanitizedBridgesWriter extends Thread {
     // Finish writing sanitized bridge descriptors to disk
     this.finishWriting();
 
+    this.checkStaleDescriptors();
+
     this.cleanUpRsyncDirectory();
   }
 
@@ -368,6 +370,8 @@ public class SanitizedBridgesWriter extends Thread {
     return this.secretsForHashingIPAddresses.get(month);
   }
 
+  private String maxNetworkStatusPublishedTime = "1970-01-01 00:00:00";
+
   /**
    * Sanitizes a network status and writes it to disk.
    */
@@ -380,6 +384,10 @@ public class SanitizedBridgesWriter extends Thread {
       return;
     }
 
+    if (publicationTime.compareTo(maxNetworkStatusPublishedTime) > 0) {
+      maxNetworkStatusPublishedTime = publicationTime;
+    }
+
     if (this.bridgeSanitizingCutOffTimestamp.
         compareTo(publicationTime) > 0) {
       this.logger.log(!this.haveWarnedAboutInterval ? Level.WARNING
@@ -543,6 +551,8 @@ public class SanitizedBridgesWriter extends Thread {
     }
   }
 
+  private String maxServerDescriptorPublishedTime = "1970-01-01 00:00:00";
+
   /**
    * Sanitizes a bridge server descriptor and writes it to disk.
    */
@@ -590,6 +600,9 @@ public class SanitizedBridgesWriter extends Thread {
          * sanitizing interval. */
         } else if (line.startsWith("published ")) {
           published = line.substring("published ".length());
+          if (published.compareTo(maxServerDescriptorPublishedTime) > 0) {
+            maxServerDescriptorPublishedTime = published;
+          }
           if (this.bridgeSanitizingCutOffTimestamp.
               compareTo(published) > 0) {
             this.logger.log(!this.haveWarnedAboutInterval
@@ -799,6 +812,9 @@ public class SanitizedBridgesWriter extends Thread {
     }
   }
 
+  private String maxExtraInfoDescriptorPublishedTime =
+      "1970-01-01 00:00:00";
+
   /**
    * Sanitizes an extra-info descriptor and writes it to disk.
    */
@@ -827,6 +843,10 @@ public class SanitizedBridgesWriter extends Thread {
         } else if (line.startsWith("published ")) {
           scrubbed.append(line + "\n");
           published = line.substring("published ".length());
+          if (published.compareTo(maxExtraInfoDescriptorPublishedTime)
+              > 0) {
+            maxExtraInfoDescriptorPublishedTime = published;
+          }
 
         /* Remove everything from transport lines except the transport
          * name. */
@@ -968,6 +988,44 @@ public class SanitizedBridgesWriter extends Thread {
     }
   }
 
+  private void checkStaleDescriptors() {
+    SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    long tooOldMillis = System.currentTimeMillis() - 330L * 60L * 1000L;
+    try {
+      long maxNetworkStatusPublishedMillis =
+          dateTimeFormat.parse(maxNetworkStatusPublishedTime).getTime();
+      if (maxNetworkStatusPublishedMillis > 0L &&
+          maxNetworkStatusPublishedMillis < tooOldMillis) {
+        this.logger.warning("The last known bridge network status was "
+            + "published " + maxNetworkStatusPublishedTime + ", which is "
+            + "more than 5:30 hours in the past.");
+      }
+      long maxServerDescriptorPublishedMillis =
+          dateTimeFormat.parse(maxServerDescriptorPublishedTime).
+          getTime();
+      if (maxServerDescriptorPublishedMillis > 0L &&
+          maxServerDescriptorPublishedMillis < tooOldMillis) {
+        this.logger.warning("The last known bridge server descriptor was "
+            + "published " + maxServerDescriptorPublishedTime + ", which "
+            + "is more than 5:30 hours in the past.");
+      }
+      long maxExtraInfoDescriptorPublishedMillis =
+          dateTimeFormat.parse(maxExtraInfoDescriptorPublishedTime).
+          getTime();
+      if (maxExtraInfoDescriptorPublishedMillis > 0L &&
+          maxExtraInfoDescriptorPublishedMillis < tooOldMillis) {
+        this.logger.warning("The last known bridge extra-info descriptor "
+            + "was published " + maxExtraInfoDescriptorPublishedTime
+            + ", which is more than 5:30 hours in the past.");
+      }
+    } catch (ParseException e) {
+      this.logger.log(Level.WARNING, "Unable to parse timestamp for "
+          + "stale check.", e);
+    }
+  }
+
   /* Delete all files from the rsync directory that have not been modified
    * in the last three days. */
   public void cleanUpRsyncDirectory() {
diff --git a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
index 0ac6f90..43d3427 100644
--- a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
+++ b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -77,6 +77,7 @@ public class BridgePoolAssignmentsProcessor extends Thread {
     SimpleDateFormat filenameFormat =
         new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
     filenameFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    long maxBridgePoolAssignmentTime = 0L;
     for (File assignmentFile : assignmentFiles) {
       logger.info("Processing bridge pool assignment file '"
           + assignmentFile.getAbsolutePath() + "'...");
@@ -120,6 +121,9 @@ public class BridgePoolAssignmentsProcessor extends Thread {
                 long bridgePoolAssignmentTime = assignmentFormat.parse(
                     bridgePoolAssignmentLine.substring(
                     "bridge-pool-assignment ".length())).getTime();
+                maxBridgePoolAssignmentTime = Math.max(
+                    maxBridgePoolAssignmentTime,
+                    bridgePoolAssignmentTime);
                 File tarballFile = new File(
                     sanitizedAssignmentsDirectory, filenameFormat.format(
                     bridgePoolAssignmentTime));
@@ -192,6 +196,18 @@ public class BridgePoolAssignmentsProcessor extends Thread {
       }
     }
 
+    if (maxBridgePoolAssignmentTime > 0L &&
+        maxBridgePoolAssignmentTime + 330L * 60L * 1000L
+        < System.currentTimeMillis()) {
+      SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+          "yyyy-MM-dd HH:mm:ss");
+      dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+      logger.warning("The last known bridge pool assignment list was "
+          + "published at "
+          + dateTimeFormat.format(maxBridgePoolAssignmentTime)
+          + ", which is more than 5:30 hours in the past.");
+    }
+
     this.cleanUpRsyncDirectory();
 
     logger.info("Finished processing bridge pool assignment file(s).");
diff --git a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
index 26e944c..9b1f40b 100644
--- a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
+++ b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
@@ -12,6 +12,7 @@ import java.net.URL;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.List;
 import java.util.SortedSet;
 import java.util.Stack;
 import java.util.TimeZone;
@@ -19,6 +20,12 @@ import java.util.TreeSet;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorParser;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.ExitList;
+import org.torproject.descriptor.ExitListEntry;
+import org.torproject.descriptor.impl.DescriptorParseException;
 import org.torproject.ernie.db.main.Configuration;
 
 public class ExitListDownloader extends Thread {
@@ -35,8 +42,19 @@ public class ExitListDownloader extends Thread {
     }
 
     Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
+
+    SimpleDateFormat dateTimeFormat =
+        new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+    Date downloadedDate = new Date();
+    String downloadedExitList = null;
     try {
       logger.fine("Downloading exit list...");
+      StringBuilder sb = new StringBuilder();
+      sb.append("@type tordnsel 1.0\n");
+      sb.append("Downloaded " + dateTimeFormat.format(downloadedDate)
+          + "\n");
       String exitAddressesUrl =
           "http://exitlist.torproject.org/exit-addresses";;
       URL u = new URL(exitAddressesUrl);
@@ -51,42 +69,72 @@ public class ExitListDownloader extends Thread {
       }
       BufferedInputStream in = new BufferedInputStream(
           huc.getInputStream());
-      SimpleDateFormat printFormat =
-          new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
-      printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-      Date downloadedDate = new Date();
-      File tarballFile = new File("exitlist/" + printFormat.format(
-          downloadedDate));
-      tarballFile.getParentFile().mkdirs();
-      File rsyncFile = new File("rsync/exit-lists/"
-          + tarballFile.getName());
-      rsyncFile.getParentFile().mkdirs();
-      SimpleDateFormat dateTimeFormat =
-          new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-      BufferedWriter bwT = new BufferedWriter(new FileWriter(
-          tarballFile));
-      BufferedWriter bwR = new BufferedWriter(new FileWriter(
-          rsyncFile));
-      bwT.write("@type tordnsel 1.0\n");
-      bwT.write("Downloaded " + dateTimeFormat.format(downloadedDate)
-          + "\n");
-      bwR.write("@type tordnsel 1.0\n");
-      bwR.write("Downloaded " + dateTimeFormat.format(downloadedDate)
-          + "\n");
       int len;
       byte[] data = new byte[1024];
       while ((len = in.read(data, 0, 1024)) >= 0) {
-        bwT.write(new String(data, 0, len));
-        bwR.write(new String(data, 0, len));
+        sb.append(new String(data, 0, len));
       }   
       in.close();
-      bwT.close();
-      bwR.close();
+      downloadedExitList = sb.toString();
       logger.fine("Finished downloading exit list.");
     } catch (IOException e) {
       logger.log(Level.WARNING, "Failed downloading exit list", e);
       return;
     }
+    if (downloadedExitList == null) {
+      logger.warning("Failed downloading exit list");
+      return;
+    }
+
+    SimpleDateFormat tarballFormat =
+        new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+    tarballFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    File tarballFile = new File("exitlist/" + tarballFormat.format(
+        downloadedDate));
+
+    long maxScanMillis = 0L;
+    try {
+      DescriptorParser descriptorParser =
+          DescriptorSourceFactory.createDescriptorParser();
+      List<Descriptor> parsedDescriptors =
+          descriptorParser.parseDescriptors(downloadedExitList.getBytes(),
+          tarballFile.getName());
+      if (parsedDescriptors.size() != 1 ||
+          !(parsedDescriptors.get(0) instanceof ExitList)) {
+        logger.warning("Could not parse downloaded exit list");
+        return;
+      }
+      ExitList parsedExitList = (ExitList) parsedDescriptors.get(0);
+      for (ExitListEntry entry : parsedExitList.getExitListEntries()) {
+        maxScanMillis = Math.max(maxScanMillis, entry.getScanMillis());
+      }
+    } catch (DescriptorParseException e) {
+      logger.log(Level.WARNING, "Could not parse downloaded exit list",
+          e);
+    }
+    if (maxScanMillis > 0L &&
+        maxScanMillis + 330L * 60L * 1000L < System.currentTimeMillis()) {
+      logger.warning("The last reported scan in the downloaded exit list "
+          + "took place at " + dateTimeFormat.format(maxScanMillis)
+          + ", which is more than 5:30 hours in the past.");
+  }
+
+    /* Write to disk. */
+    File rsyncFile = new File("rsync/exit-lists/"
+        + tarballFile.getName());
+    File[] outputFiles = new File[] { tarballFile, rsyncFile };
+    for (File outputFile : outputFiles) {
+      try {
+        outputFile.getParentFile().mkdirs();
+        BufferedWriter bw = new BufferedWriter(new FileWriter(
+            outputFile));
+        bw.write(downloadedExitList);
+        bw.close();
+      } catch (IOException e) {
+        logger.log(Level.WARNING, "Could not write downloaded exit list "
+            + "to " + outputFile.getAbsolutePath(), e);
+      }
+    }
 
     /* Write stats. */
     StringBuilder dumpStats = new StringBuilder("Finished downloading "
diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
index f95bbf7..9a07ada 100644
--- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
+++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
@@ -94,6 +94,8 @@ public class ArchiveWriter extends Thread {
     // Write output to disk that only depends on relay descriptors
     this.dumpStats();
 
+    this.checkStaledescriptors();
+
     this.cleanUpRsyncDirectory();
   }
 
@@ -128,9 +130,12 @@ public class ArchiveWriter extends Thread {
     return false;
   }
 
+  private long maxConsensusValidAfter = 0L;
   private static final byte[] CONSENSUS_ANNOTATION =
       "@type network-status-consensus-3 1.0\n".getBytes();
   public void storeConsensus(byte[] data, long validAfter) {
+    this.maxConsensusValidAfter = Math.max(this.maxConsensusValidAfter,
+        validAfter);
     SimpleDateFormat printFormat = new SimpleDateFormat(
         "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
     printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -144,10 +149,12 @@ public class ArchiveWriter extends Thread {
     }
   }
 
+  private long maxVoteValidAfter = 0L;
   private static final byte[] VOTE_ANNOTATION =
       "@type network-status-vote-3 1.0\n".getBytes();
   public void storeVote(byte[] data, long validAfter,
       String fingerprint, String digest) {
+    this.maxVoteValidAfter = Math.max(this.maxVoteValidAfter, validAfter);
     SimpleDateFormat printFormat = new SimpleDateFormat(
         "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
     printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -177,10 +184,13 @@ public class ArchiveWriter extends Thread {
     }
   }
 
+  private long maxServerDescriptorPublished = 0L;
   private static final byte[] SERVER_DESCRIPTOR_ANNOTATION =
       "@type server-descriptor 1.0\n".getBytes();
   public void storeServerDescriptor(byte[] data, String digest,
       long published) {
+    this.maxServerDescriptorPublished = Math.max(
+        this.maxServerDescriptorPublished, published);
     SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
     printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
     File tarballFile = new File(this.outputDirectory
@@ -195,10 +205,13 @@ public class ArchiveWriter extends Thread {
     }
   }
 
+  private long maxExtraInfoDescriptorPublished = 0L;
   private static final byte[] EXTRA_INFO_ANNOTATION =
       "@type extra-info 1.0\n".getBytes();
   public void storeExtraInfoDescriptor(byte[] data,
       String extraInfoDigest, long published) {
+    this.maxExtraInfoDescriptorPublished = Math.max(
+        this.maxExtraInfoDescriptorPublished, published);
     SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
     descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
     File tarballFile = new File(this.outputDirectory + "/extra-info/"
@@ -403,6 +416,39 @@ public class ArchiveWriter extends Thread {
     }
   }
 
+  private void checkStaledescriptors() {
+    SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    long tooOldMillis = System.currentTimeMillis() - 330L * 60L * 1000L;
+    if (maxConsensusValidAfter > 0L &&
+        maxConsensusValidAfter < tooOldMillis) {
+      this.logger.warning("The last known relay network status "
+          + "consensus was valid after "
+          + dateTimeFormat.format(maxConsensusValidAfter)
+          + ", which is more than 5:30 hours in the past.");
+    }
+    if (maxVoteValidAfter > 0L && maxVoteValidAfter < tooOldMillis) {
+      this.logger.warning("The last known relay network status vote "
+          + "was valid after " + dateTimeFormat.format(maxVoteValidAfter)
+          + ", which is more than 5:30 hours in the past.");
+    }
+    if (maxServerDescriptorPublished > 0L &&
+        maxServerDescriptorPublished < tooOldMillis) {
+      this.logger.warning("The last known relay server descriptor was "
+          + "published at "
+          + dateTimeFormat.format(maxServerDescriptorPublished)
+          + ", which is more than 5:30 hours in the past.");
+    }
+    if (maxExtraInfoDescriptorPublished > 0L &&
+        maxExtraInfoDescriptorPublished < tooOldMillis) {
+      this.logger.warning("The last known relay extra-info descriptor "
+          + "was published at "
+          + dateTimeFormat.format(maxExtraInfoDescriptorPublished)
+          + ", which is more than 5:30 hours in the past.");
+    }
+  }
+
   /* Delete all files from the rsync directory that have not been modified
    * in the last three days. */
   public void cleanUpRsyncDirectory() {

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits