[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] [ernie/master 3/3] Add useful logs about the last execution to website.



Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed, 14 Apr 2010 12:22:53 +0200
Subject: Add useful logs about the last execution to website.
Commit: ac61e609853e63669c84fe31bff9c667a5cf455d

---
 src/ArchiveReader.java               |    6 ++
 src/ArchiveWriter.java               |  123 +++++++++++++++++++---------------
 src/CachedRelayDescriptorReader.java |   11 +++
 src/Main.java                        |    9 ++-
 src/RelayDescriptorDownloader.java   |   79 ++++++++++++++++++++++
 5 files changed, 169 insertions(+), 59 deletions(-)

diff --git a/src/ArchiveReader.java b/src/ArchiveReader.java
index 92a07aa..ef68cc1 100644
--- a/src/ArchiveReader.java
+++ b/src/ArchiveReader.java
@@ -10,6 +10,7 @@ import org.apache.commons.compress.compressors.bzip2.*;
 public class ArchiveReader {
   public ArchiveReader(RelayDescriptorParser rdp, String archivesDir,
       boolean keepImportHistory) {
+    int parsedFiles = 0, ignoredFiles = 0;
     Logger logger = Logger.getLogger(ArchiveReader.class.getName());
     SortedSet<String> archivesImportHistory = new TreeSet<String>();
     File archivesImportHistoryFile =
@@ -46,6 +47,7 @@ public class ArchiveReader {
               BufferedInputStream bis = null;
               if (keepImportHistory &&
                   archivesImportHistory.contains(pop.getName())) {
+                ignoredFiles++;
                 continue;
               } else if (pop.getName().endsWith(".tar.bz2")) {
                 logger.warning("Cannot parse compressed tarball "
@@ -72,6 +74,7 @@ public class ArchiveReader {
               bis.close();
               byte[] allData = baos.toByteArray();
               rdp.parse(allData);
+              parsedFiles++;
             } catch (IOException e) {
               problems.add(pop);
               if (problems.size() > 3) {
@@ -111,6 +114,9 @@ public class ArchiveReader {
             + "history file.");
       }
     }
+    logger.info("Finished importing relay descriptors from local "
+        + "directory:\nParsed " + parsedFiles + " and ignored "
+        + ignoredFiles + " files.");
   }
 }
 
diff --git a/src/ArchiveWriter.java b/src/ArchiveWriter.java
index de72da1..051431f 100644
--- a/src/ArchiveWriter.java
+++ b/src/ArchiveWriter.java
@@ -8,12 +8,15 @@ import org.apache.commons.codec.binary.*;
 public class ArchiveWriter {
   private Logger logger;
   private String outputDirectory;
+  private int storedConsensuses = 0, storedVotes = 0,
+      storedServerDescriptors = 0, storedExtraInfoDescriptors = 0;
+
   public ArchiveWriter(String outputDirectory) {
     this.logger = Logger.getLogger(ArchiveWriter.class.getName());
     this.outputDirectory = outputDirectory;
   }
 
-  private void store(byte[] data, String filename) {
+  private boolean store(byte[] data, String filename) {
     try {
       File file = new File(filename);
       if (!file.exists()) {
@@ -23,11 +26,13 @@ public class ArchiveWriter {
             new FileOutputStream(file));
         bos.write(data, 0, data.length);
         bos.close();
+        return true;
       }
     } catch (IOException e) {
       this.logger.log(Level.WARNING, "Could not store relay descriptor "
           + filename, e);
     }
+    return false;
   }
 
   public void storeConsensus(byte[] data, long validAfter) {
@@ -36,7 +41,9 @@ public class ArchiveWriter {
     printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
     String filename = outputDirectory + "/consensus/"
         + printFormat.format(new Date(validAfter)) + "-consensus";
-    this.store(data, filename);
+    if (this.store(data, filename)) {
+      this.storedConsensuses++;
+    }
   }
 
   public void storeVote(byte[] data, long validAfter,
@@ -47,7 +54,9 @@ public class ArchiveWriter {
     String filename = outputDirectory + "/vote/"
         + printFormat.format(new Date(validAfter)) + "-vote-"
         + fingerprint + "-" + digest;
-    this.store(data, filename);
+    if (this.store(data, filename)) {
+      this.storedVotes++;
+    }
   }
 
   public void storeServerDescriptor(byte[] data, String digest,
@@ -58,7 +67,9 @@ public class ArchiveWriter {
         + printFormat.format(new Date(published))
         + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
         + digest;
-    this.store(data, filename);
+    if (this.store(data, filename)) {
+      this.storedServerDescriptors++;
+    }
   }
 
   public void storeExtraInfoDescriptor(byte[] data,
@@ -70,27 +81,9 @@ public class ArchiveWriter {
         + extraInfoDigest.substring(0, 1) + "/"
         + extraInfoDigest.substring(1, 2) + "/"
         + extraInfoDigest;
-    this.store(data, filename);
-  }
-
-  private SortedSet<String> getFileNames(File dir) {
-    SortedSet<String> files = new TreeSet<String>();
-    Stack<File> leftToParse = new Stack<File>();
-    leftToParse.add(dir);
-    while (!leftToParse.isEmpty()) {
-      File pop = leftToParse.pop();
-      if (pop.isDirectory()) {
-        for (File f : pop.listFiles()) {
-          leftToParse.add(f);
-        }
-      } else if (pop.length() > 0) {
-        String absPath = pop.getAbsolutePath().replaceAll(":", "-");
-        String relPath = absPath.substring(absPath.indexOf(
-            outputDirectory + "/"));
-        files.add(relPath);
-      }
+    if (this.store(data, filename)) {
+      this.storedExtraInfoDescriptors++;
     }
-    return files;
   }
 
   /**
@@ -98,15 +91,16 @@ public class ArchiveWriter {
    * on level INFO.
    */
   public void dumpStats() {
+    StringBuilder sb = new StringBuilder("Finished writing relay "
+        + "descriptors to disk:\nIn this execution, we stored "
+        + this.storedConsensuses + " consensuses, " + this.storedVotes
+        + " votes, " + this.storedServerDescriptors
+        + " server descriptors, and " + this.storedExtraInfoDescriptors
+        + " extra-info descriptors to disk.\n");
+    sb.append("Statistics on the completeness of written relay "
+        + "descriptors of the past 12 consensuses (Consensus/Vote, "
+        + "valid-after, votes, server descriptors, extra-infos):");
     try {
-      SortedSet<String> votes = getFileNames(
-          new File(outputDirectory + "/vote"));
-      SortedSet<String> serverDescs = getFileNames(
-          new File(outputDirectory + "/server-descriptor"));
-      SortedSet<String> extraInfos = getFileNames(
-          new File(outputDirectory + "/extra-info"));
-      SortedSet<String> consensuses = getFileNames(
-          new File(outputDirectory + "/consensus"));
       SimpleDateFormat validAfterFormat =
           new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
       validAfterFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -116,17 +110,37 @@ public class ArchiveWriter {
       SimpleDateFormat descriptorFormat =
           new SimpleDateFormat("yyyy/MM/");
       descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-      StringBuilder sb = new StringBuilder();
-      sb.append("  valid-after          votes         "
-          + "server descriptors  extra-infos\n");
-      SortedSet<String> lastConsensuses = new TreeSet<String>();
-      for (int i = 0; !consensuses.isEmpty() && i < 12; i++) {
-        String last = consensuses.last();
-        lastConsensuses.add(last);
-        consensuses.remove(last);
+
+      SortedSet<File> consensuses = new TreeSet<File>();
+      Stack<File> leftToParse = new Stack<File>();
+      leftToParse.add(new File(outputDirectory + "/consensus"));
+      while (!leftToParse.isEmpty()) {
+        File pop = leftToParse.pop();
+        if (pop.isDirectory()) {
+          for (File f : pop.listFiles()) {
+            leftToParse.add(f);
+          }
+        } else if (pop.length() > 0) {
+          consensuses.add(pop);
+        }
+        while (consensuses.size() > 12) {
+          consensuses.remove(consensuses.first());
+        }
+      }
+      leftToParse.add(new File(outputDirectory + "/vote"));
+      SortedSet<File> votes = new TreeSet<File>();
+      while (!leftToParse.isEmpty()) {
+        File pop = leftToParse.pop();
+        if (pop.isDirectory()) {
+          for (File f : pop.listFiles()) {
+            leftToParse.add(f);
+          }
+        } else if (pop.length() > 0) {
+          votes.add(pop);
+        }
       }
-      for (String f : lastConsensuses) {
-        BufferedReader br = new BufferedReader(new FileReader(new File(f)));
+      for (File f : consensuses) {
+        BufferedReader br = new BufferedReader(new FileReader(f));
         String line = null, validAfterTime = null, votePrefix = null;
         int allVotes = 0, foundVotes = 0,
             allServerDescs = 0, foundServerDescs = 0,
@@ -143,9 +157,9 @@ public class ArchiveWriter {
             allVotes++;
             String pattern = votePrefix + line.split(" ")[2];
             String votefilename = null;
-            for (String v : votes) {
-              if (v.startsWith(pattern)) {
-                votefilename = v;
+            for (File v : votes) {
+              if (v.getName().startsWith(pattern)) {
+                votefilename = v.getName();
                 break;
               }
             }
@@ -169,7 +183,7 @@ public class ArchiveWriter {
                       + descriptorFormat.format(new Date(published))
                       + digest.substring(0, 1) + "/"
                       + digest.substring(1, 2) + "/" + digest;
-                  if (serverDescs.contains(filename)) {
+                  if (new File(filename).exists()) {
                     BufferedReader sbr = new BufferedReader(new FileReader(
                         new File(filename)));
                     String line2 = null;
@@ -186,7 +200,7 @@ public class ArchiveWriter {
                             + extraInfoDigest.substring(0, 1) + "/"
                             + extraInfoDigest.substring(1, 2) + "/"
                             + extraInfoDigest;
-                        if (extraInfos.contains(filename2)) {
+                        if (new File(filename2).exists()) {
                           voteFoundExtraInfos++;
                         }
                       }
@@ -197,8 +211,8 @@ public class ArchiveWriter {
                 }
               }
               vbr.close();
-              sb.append(String.format("V %s               "
-                  + " %d/%d (%5.1f%%)  %d/%d (%5.1f%%)%n",
+              sb.append(String.format("%nV %s               "
+                  + " %d/%d (%5.1f%%)  %d/%d (%5.1f%%)",
                   validAfterTime,
                   voteFoundServerDescs, voteAllServerDescs,
                   100.0D * (double) voteFoundServerDescs /
@@ -217,7 +231,7 @@ public class ArchiveWriter {
                 + descriptorFormat.format(new Date(published))
                 + digest.substring(0, 1) + "/"
                 + digest.substring(1, 2) + "/" + digest;
-            if (serverDescs.contains(filename)) {
+            if (new File (filename).exists()) {
               BufferedReader sbr = new BufferedReader(new FileReader(
                   new File(filename)));
               String line2 = null;
@@ -233,7 +247,7 @@ public class ArchiveWriter {
                       + extraInfoDigest.substring(0, 1) + "/"
                       + extraInfoDigest.substring(1, 2) + "/"
                       + extraInfoDigest;
-                  if (extraInfos.contains(filename2)) {
+                  if (new File (filename2).exists()) {
                     foundExtraInfos++;
                   }
                 }
@@ -243,8 +257,8 @@ public class ArchiveWriter {
             }
           }
         }
-        sb.append(String.format("C %s  %d/%d (%5.1f%%)  %d/%d (%5.1f%%)  "
-            + "%d/%d (%5.1f%%)%n",
+        sb.append(String.format("%nC %s  %d/%d (%5.1f%%)  %d/%d (%5.1f%%)  "
+            + "%d/%d (%5.1f%%)",
             validAfterTime, foundVotes, allVotes,
             100.0D * (double) foundVotes / (double) allVotes,
             foundServerDescs, allServerDescs,
@@ -252,8 +266,7 @@ public class ArchiveWriter {
             foundExtraInfos, allExtraInfos,
             100.0D * (double) foundExtraInfos / (double) allExtraInfos));
       }
-      this.logger.info("Statistics on relay descriptors from the last 12 "
-          + "known consensuses:\n" + sb.toString());
+      this.logger.info(sb.toString());
     } catch (IOException e) {
       this.logger.log(Level.WARNING, "Could not dump statistics to disk.",
           e);
diff --git a/src/CachedRelayDescriptorReader.java b/src/CachedRelayDescriptorReader.java
index 64a1966..cb5ca28 100644
--- a/src/CachedRelayDescriptorReader.java
+++ b/src/CachedRelayDescriptorReader.java
@@ -10,6 +10,8 @@ import java.util.logging.*;
 public class CachedRelayDescriptorReader {
   public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
       List<String> inputDirectories) {
+    StringBuilder dumpStats = new StringBuilder("Finished importing "
+        + "relay descriptors from local Tor data directories:");
     Logger logger = Logger.getLogger(
         CachedRelayDescriptorReader.class.getName());
     for (String inputDirectory : inputDirectories) {
@@ -42,6 +44,8 @@ public class CachedRelayDescriptorReader {
             String line = null;
             while ((line = br.readLine()) != null) {
               if (line.startsWith("valid-after ")) {
+                dumpStats.append("\n" + f.getName() + ": " + line.substring(
+                    "valid-after ".length()));
                 SimpleDateFormat dateTimeFormat =
                     new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                 dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -52,6 +56,7 @@ public class CachedRelayDescriptorReader {
                       + cachedDescDir.getAbsolutePath() + " are stale. "
                       + "The valid-after line in cached-consensus is '"
                       + line + "'.");
+                  dumpStats.append(" (stale!)");
                 }
                 break;
               }
@@ -72,6 +77,7 @@ public class CachedRelayDescriptorReader {
                 "router " : "extra-info ";
             String sigToken = "\nrouter-signature\n";
             String endToken = "\n-----END SIGNATURE-----\n";
+            int parsedNum = 0;
             while (end < ascii.length()) {
               start = ascii.indexOf(startToken, end);
               if (start < 0) {
@@ -91,8 +97,12 @@ public class CachedRelayDescriptorReader {
               System.arraycopy(allData, start, descBytes, 0, end - start);
               if (rdp != null) {
                 rdp.parse(descBytes);
+                parsedNum++;
               }
             }
+            dumpStats.append("\n" + f.getName() + ": " + parsedNum
+                + (f.getName().startsWith("cached-descriptors") ?
+                "server" : "extra-info") + " descriptors");
             logger.fine("Finished reading "
                 + cachedDescDir.getAbsolutePath() + " directory.");
           }
@@ -105,6 +115,7 @@ public class CachedRelayDescriptorReader {
         }
       }
     }
+    logger.info(dumpStats.toString());
   }
 }
 
diff --git a/src/Main.java b/src/Main.java
index d1e4c6a..a147cc9 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -92,10 +92,15 @@ public class Main {
       if (rdd != null) {
         rdd.downloadMissingDescriptors();
         rdd.writeFile();
+        rdd = null;
       }
     }
 
     // Write output to disk that only depends on relay descriptors
+    if (aw != null) {
+      aw.dumpStats();
+      aw = null;
+    }
     if (dsfh != null) {
       dsfh.writeFile();
       dsfh = null;
@@ -104,10 +109,6 @@ public class Main {
       sdsfh.writeFiles();
       sdsfh = null;
     }
-    if (aw != null) {
-      aw.dumpStats();
-      aw = null;
-    }
 
     // Import/download GeoIP databases
     GeoIPDatabaseManager gd = new GeoIPDatabaseManager(
diff --git a/src/RelayDescriptorDownloader.java b/src/RelayDescriptorDownloader.java
index d7654c5..53ce5c1 100644
--- a/src/RelayDescriptorDownloader.java
+++ b/src/RelayDescriptorDownloader.java
@@ -99,6 +99,13 @@ public class RelayDescriptorDownloader {
    */
   private Logger logger;
 
+  private StringBuilder dumpStats;
+  private int newMissingConsensuses = 0, newMissingVotes = 0,
+      newMissingServerDescriptors = 0, newMissingExtraInfoDescriptors = 0,
+      triedConsensuses = 0, triedVotes = 0, triedServerDescriptors = 0,
+      triedExtraInfoDescriptors = 0, downloadedConsensuses = 0,
+      downloadedVotes = 0, downloadedServerDescriptors = 0,
+      downloadedExtraInfoDescriptors = 0;
   /**
    * Initializes this class, including reading in missing descriptors from
    * <code>stats/missing-relay-descriptors</code>.
@@ -144,6 +151,8 @@ public class RelayDescriptorDownloader {
      * directory servers. */
     this.missingDescriptorsFile = new File(
         "stats/missing-relay-descriptors");
+    int missingConsensuses = 0, missingVotes = 0,
+        missingServerDescriptors = 0, missingExtraInfoDescriptors = 0;
     if (this.missingDescriptorsFile.exists()) {
       try {
         this.logger.fine("Reading file "
@@ -160,6 +169,15 @@ public class RelayDescriptorDownloader {
                 ((line.startsWith("server,") ||
                 line.startsWith("extra,")) &&
                 this.descriptorCutOff.compareTo(published) <= 0)) {
+              if (line.startsWith("consensus,")) {
+                missingConsensuses++;
+              } else if (line.startsWith("vote,")) {
+                missingVotes++;
+              } else if (line.startsWith("server,")) {
+                missingServerDescriptors++;
+              } else if (line.startsWith("extra,")) {
+                missingExtraInfoDescriptors++;
+              }
               int separateAt = line.lastIndexOf(",");
               this.missingDescriptors.put(line.substring(0,
                   separateAt), line.substring(separateAt + 1));
@@ -179,6 +197,13 @@ public class RelayDescriptorDownloader {
             + "! This means that we might forget to dowload relay "
             + "descriptors we are missing.", e);
       }
+
+      dumpStats.append("Finished downloading relay descriptors from the "
+        + "directory authorities:\nAt the beginning of this execution, "
+        + "we were missing " + missingConsensuses + " consensuses, "
+        + missingVotes + " votes, " + missingServerDescriptors
+        + " server descriptors, and " + missingExtraInfoDescriptors
+        + " extra-info descriptors.");
     }
   }
 
@@ -203,6 +228,7 @@ public class RelayDescriptorDownloader {
           String voteKey = "vote," + validAfter + "," + dirSource;
           if (!this.missingDescriptors.containsKey(voteKey)) {
             this.missingDescriptors.put(voteKey, "NA");
+            this.newMissingVotes++;
           }
         }
       }
@@ -222,6 +248,7 @@ public class RelayDescriptorDownloader {
             if (!this.missingDescriptors.containsKey(
                 serverDescriptorKey)) {
               this.missingDescriptors.put(serverDescriptorKey, "NA");
+              this.newMissingServerDescriptors++;
             }
           }
         }
@@ -254,6 +281,7 @@ public class RelayDescriptorDownloader {
             if (!this.missingDescriptors.containsKey(
                 serverDescriptorKey)) {
               this.missingDescriptors.put(serverDescriptorKey, "NA");
+              this.newMissingServerDescriptors++;
             }
           }
         }
@@ -286,6 +314,7 @@ public class RelayDescriptorDownloader {
             + relayIdentity + "," + extraInfoDigest;
         if (!this.missingDescriptors.containsKey(extraInfoKey)) {
           this.missingDescriptors.put(extraInfoKey, "NA");
+          this.newMissingExtraInfoDescriptors++;
         }
       }
     }
@@ -350,10 +379,12 @@ public class RelayDescriptorDownloader {
               this.downloadCurrentConsensus &&
               this.currentValidAfter.equals(parts[1])) {
             urls.add("/tor/status-vote/current/consensus");
+            this.triedConsensuses++;
           } else if (parts[0].equals("vote") &&
               this.downloadCurrentVotes &&
               this.currentValidAfter.equals(parts[1])) {
             urls.add("/tor/status-vote/current/" + parts[2]);
+            this.triedVotes++;
           } else if (parts[0].equals("server") &&
               (this.downloadAllServerDescriptors ||
               (this.downloadDescriptorsForRelays != null &&
@@ -361,6 +392,7 @@ public class RelayDescriptorDownloader {
               toUpperCase()))) &&
               this.descriptorCutOff.compareTo(parts[1]) <= 0) {
             urls.add("/tor/server/d/" + parts[3]);
+            this.triedServerDescriptors++;
           } else if (parts[0].equals("extra") &&
               (this.downloadAllExtraInfos ||
               (this.downloadDescriptorsForRelays != null &&
@@ -368,6 +400,7 @@ public class RelayDescriptorDownloader {
               toUpperCase()))) &&
               this.descriptorCutOff.compareTo(parts[1]) <= 0) {
             urls.add("/tor/extra/d/" + parts[3]);
+            this.triedExtraInfoDescriptors++;
           }
         }
       }
@@ -413,6 +446,15 @@ public class RelayDescriptorDownloader {
             in.close();
             byte[] allData = baos.toByteArray();
             rdp.parse(allData);
+            if (url.endsWith("consensus")) {
+              this.downloadedConsensuses++;
+            } else if (url.contains("status-vote")) {
+              this.downloadedVotes++;
+            } else if (url.contains("server")) {
+              this.downloadedServerDescriptors++;
+            } else if (url.contains("extra")) {
+              this.downloadedExtraInfoDescriptors++;
+            }
           } else {
             retryUrls.add(url);
           }
@@ -440,6 +482,8 @@ public class RelayDescriptorDownloader {
   }
 
   public void writeFile() {
+    int missingConsensuses = 0, missingVotes = 0,
+        missingServerDescriptors = 0, missingExtraInfoDescriptors = 0;
     try {
       this.logger.fine("Writing file "
           + this.missingDescriptorsFile.getAbsolutePath() + "...");
@@ -448,6 +492,16 @@ public class RelayDescriptorDownloader {
           this.missingDescriptorsFile));
       for (Map.Entry<String, String> e :
           this.missingDescriptors.entrySet()) {
+        String key = e.getKey();
+        if (key.startsWith("consensus,")) {
+          missingConsensuses++;
+        } else if (key.startsWith("vote,")) {
+          missingVotes++;
+        } else if (key.startsWith("server,")) {
+          missingServerDescriptors++;
+        } else if (key.startsWith("extra,")) {
+          missingExtraInfoDescriptors++;
+        }
         bw.write(e.getKey() + "," + e.getValue() + "\n");
       }
       bw.close();
@@ -457,5 +511,30 @@ public class RelayDescriptorDownloader {
       this.logger.log(Level.WARNING, "Failed writing "
           + this.missingDescriptorsFile.getAbsolutePath() + "!", e);
     }
+
+    dumpStats.append("During this execution, we added "
+        + this.newMissingConsensuses + " consensuses, "
+        + this.newMissingVotes + " votes, "
+        + this.newMissingServerDescriptors + " server descriptors, and "
+        + this.newMissingExtraInfoDescriptors + " extra-info descriptors "
+        + "to the missing list.\n");
+    dumpStats.append("We attempted to download " + this.triedConsensuses
+        + " consensuses, " + this.triedVotes + " votes, "
+        + this.triedServerDescriptors + " server descriptors, and "
+        + this.triedExtraInfoDescriptors + " extra-info descriptors from "
+        + "the directory authorities.\n");
+    dumpStats.append("We successfully downloaded "
+        + this.downloadedConsensuses + " consensuses, "
+        + this.downloadedVotes + " votes, "
+        + this.downloadedServerDescriptors + " server descriptors, and "
+        + this.downloadedExtraInfoDescriptors + " extra-info descriptors "
+        + "from the directory authorities.\n");
+    dumpStats.append("At the end of this execution, "
+      + "we are missing " + missingConsensuses + " consensuses, "
+      + missingVotes + " votes, " + missingServerDescriptors
+      + " server descriptors, and " + missingExtraInfoDescriptors
+      + " extra-info descriptors, some of which we may try in the next "
+      + "execution.");
+    this.logger.info(dumpStats.toString());
   }
 }
-- 
1.6.5