[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] [ernie/master] Add useful log statements for some more data sinks.



Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed, 14 Apr 2010 21:31:32 +0200
Subject: Add useful log statements for some more data sinks.
Commit: 46504e93e56922ff3f105095347d77d147bb386c

---
 src/ConsensusStatsFileHandler.java        |   11 ++++++++++
 src/DirreqStatsFileHandler.java           |   25 +++++++++++++++++++++++
 src/ExitListDownloader.java               |   31 +++++++++++++++++++++++++++++
 src/GetTorProcessor.java                  |    4 +++
 src/ServerDescriptorStatsFileHandler.java |   19 +++++++++++++++++
 src/TorperfProcessor.java                 |   26 ++++++++++++++++++++++++
 6 files changed, 116 insertions(+), 0 deletions(-)

diff --git a/src/ConsensusStatsFileHandler.java b/src/ConsensusStatsFileHandler.java
index ceb7a48..906a376 100644
--- a/src/ConsensusStatsFileHandler.java
+++ b/src/ConsensusStatsFileHandler.java
@@ -82,6 +82,8 @@ public class ConsensusStatsFileHandler {
    */
   private Logger logger;
 
+  private int relayResultsAdded = 0, bridgeResultsAdded = 0;
+
  /**
   * Initializes this class, including reading in intermediate results
   * files <code>stats/consensus-stats-raw</code> and
@@ -231,6 +233,7 @@ public class ConsensusStatsFileHandler {
       this.logger.finer("Adding new relay numbers: " + line);
       this.relaysRaw.put(validAfter, line);
       this.relaysRawModified = true;
+      this.relayResultsAdded++;
     } else if (!line.equals(this.relaysRaw.get(validAfter))) {
       this.logger.warning("The numbers of relays with Exit, Fast, "
         + "Guard, Running, and Stable flag we were just given (" + line
@@ -251,6 +254,7 @@ public class ConsensusStatsFileHandler {
       this.logger.finer("Adding new bridge numbers: " + line);
       this.bridgesRaw.put(published, line);
       this.bridgesRawModified = true;
+      this.bridgeResultsAdded++;
     } else if (!line.equals(this.bridgesRaw.get(published))) {
       this.logger.warning("The numbers of running bridges we were just "
         + "given (" + line + ") are different from what we learned "
@@ -472,6 +476,13 @@ public class ConsensusStatsFileHandler {
 
     /* Set modification flags to false again. */
     this.relaysRawModified = this.bridgesRawModified = false;
+
+    logger.info("Finished writing statistics on relay consensuses and "
+        + "bridge statuses to disk.\nAdded " + this.relayResultsAdded
+        + "relay consensuses and " + this.bridgeResultsAdded + " bridge "
+        + "statuses in this execution.\nLast known relay consensus was "
+        + "published "+ this.relaysRaw.lastKey() + ", last known bridge "
+        + "status was published " + this.bridgesRaw.lastKey());
   }
 }
 
diff --git a/src/DirreqStatsFileHandler.java b/src/DirreqStatsFileHandler.java
index 17bdc19..07fcd7d 100644
--- a/src/DirreqStatsFileHandler.java
+++ b/src/DirreqStatsFileHandler.java
@@ -40,6 +40,8 @@ public class DirreqStatsFileHandler {
    */
   private Logger logger;
 
+  private int addedResults = 0;
+
   /**
    * Initializes this class, including reading in previous results from
    * <code>stats/dirreq-stats</code>.
@@ -139,6 +141,7 @@ public class DirreqStatsFileHandler {
       this.logger.finer("Adding new directory request numbers: " + value);
       this.dirreqs.put(key, value);
       this.dirreqsModified = true;
+      this.addedResults++;
     } else if (value.compareTo(this.dirreqs.get(key)) > 0) {
       this.logger.warning("The directory request numbers we were just "
           + "given (" + value + ") are different from what we learned "
@@ -213,6 +216,28 @@ public class DirreqStatsFileHandler {
 
     /* Set modification flag to false again. */
     this.dirreqsModified = false;
+
+    /* Write stats. */
+    StringBuilder dumpStats = new StringBuilder("Finished writing "
+        + "statistics on directory requests by country.\nAdded "
+        + this.addedResults + " new observations in this execution.\n"
+        + "Last known obserations by directory are:");
+    String lastDir = null;
+    String lastDate = null;
+    for (String line : this.dirreqs.keySet()) {
+      String[] parts = line.split(",");
+      if (lastDir == null) {
+        lastDir = parts[0];
+      } else if (!parts[0].equals(lastDir)) {
+        dumpStats.append("\n" + lastDir.substring(0, 8) + " " + lastDate);
+        lastDir = parts[0];
+      }
+      lastDate = parts[1];
+    }
+    if (lastDir != null) {
+      dumpStats.append("\n" + lastDir.substring(0, 8) + " " + lastDate);
+    }
+    logger.info(dumpStats.toString());
   }
 }
 
diff --git a/src/ExitListDownloader.java b/src/ExitListDownloader.java
index 629af37..f280dca 100644
--- a/src/ExitListDownloader.java
+++ b/src/ExitListDownloader.java
@@ -43,6 +43,37 @@ public class ExitListDownloader {
       logger.log(Level.WARNING, "Failed downloading exit list", e);
       return;
     }
+
+    /* Write stats. */
+    StringBuilder dumpStats = new StringBuilder("Finished downloading "
+        + "exit list.\nLast three exit lists are:");
+    Stack<File> filesInInputDir = new Stack<File>();
+    filesInInputDir.add(new File("exitlist"));
+    SortedSet<File> lastThreeExitLists = new TreeSet<File>();
+    while (!filesInInputDir.isEmpty()) {
+      File pop = filesInInputDir.pop();
+      if (pop.isDirectory()) {
+        SortedSet<File> lastThreeElements = new TreeSet<File>();
+        for (File f : pop.listFiles()) {
+          lastThreeElements.add(f);
+        }
+        while (lastThreeElements.size() > 3) {
+          lastThreeElements.remove(lastThreeElements.first());
+        }
+        for (File f : lastThreeElements) {
+          filesInInputDir.add(f);
+        }
+      } else {
+        lastThreeExitLists.add(pop);
+        while (lastThreeExitLists.size() > 3) {
+          lastThreeExitLists.remove(lastThreeExitLists.first());
+        }
+      }
+    }
+    for (File f : lastThreeExitLists) {
+      dumpStats.append("\n" + f.getName());
+    }
+    logger.info(dumpStats.toString());
   }
 }
 
diff --git a/src/GetTorProcessor.java b/src/GetTorProcessor.java
index 356b5a7..00abe5f 100644
--- a/src/GetTorProcessor.java
+++ b/src/GetTorProcessor.java
@@ -83,6 +83,10 @@ public class GetTorProcessor {
       logger.log(Level.WARNING, "Failed writing "
           + statsFile.getAbsolutePath() + "!", e);
     }
+
+    logger.info("Finished downloading and processing statistics on Tor "
+        + "packages delivered by GetTor.\nDownloaded " + unparsed.length()
+        + " bytes. Last date in statistics is " + data.lastKey());
   }
 }
 
diff --git a/src/ServerDescriptorStatsFileHandler.java b/src/ServerDescriptorStatsFileHandler.java
index f993ddb..e1156fb 100644
--- a/src/ServerDescriptorStatsFileHandler.java
+++ b/src/ServerDescriptorStatsFileHandler.java
@@ -120,6 +120,8 @@ public class ServerDescriptorStatsFileHandler {
 
   // TODO should there be a modified flag, too?
 
+  private int addedConsensuses = 0, addedServerDescriptors = 0;
+
   /**
    * Initializes this class, without reading in any files. We're only
    * reading in files when writing results to disk in
@@ -164,6 +166,7 @@ public class ServerDescriptorStatsFileHandler {
     if (!this.consensuses.containsKey(validAfter)) {
       this.logger.finer("Adding consensus published at " + validAfter
           + ".");
+      this.addedConsensuses++;
     } else {
       this.logger.fine("We already learned about a consensus published "
           + "at " + validAfter + " in this execution. Overwriting.");
@@ -228,6 +231,7 @@ public class ServerDescriptorStatsFileHandler {
     if (!this.descriptors.containsKey(key)) {
       this.logger.finer("Adding server descriptor with identifier "
           + descriptorIdentity + ".");
+      this.addedServerDescriptors++;
     } else {
       this.logger.fine("We already learned about a server descriptor "
           + "with identifier " + descriptorIdentity + ", published at "
@@ -259,6 +263,8 @@ public class ServerDescriptorStatsFileHandler {
    */
   public void writeFiles() {
 
+   String lastWrittenDay = null;
+
    try {
 
       /* Initialize readers for reading intermediate results files from
@@ -432,6 +438,7 @@ public class ServerDescriptorStatsFileHandler {
            * least half of the possible 24 consensuses of that day, write
            * stats to disk. */
           if (consensusesAtThisDay >= 12) {
+            lastWrittenDay = statsDate;
             versionWriter.write(statsDate);
             for (int i = 0; i < versionStats.length; i++) {
               versionWriter.write("," + (versionStats[i] /
@@ -599,5 +606,17 @@ public class ServerDescriptorStatsFileHandler {
     } catch (Exception e) {
       this.logger.log(Level.WARNING, "Exception while writing files.", e);
     }
+
+    StringBuilder dumpStats = new StringBuilder("Finished writing "
+        + "statistics information contained in consensuses and server "
+        + "descriptors.\nAdded " + this.addedConsensuses + " consensuses "
+        + "and " + this.addedServerDescriptors + " in this execution.\n");
+    if (lastWrittenDay == null) {
+      dumpStats.append("No statistics written so far.");
+    } else {
+      dumpStats.append("Last written day of statistics was "
+          + lastWrittenDay);
+    }
+    this.logger.info(dumpStats.toString());
   }
 }
diff --git a/src/TorperfProcessor.java b/src/TorperfProcessor.java
index b851249..2d7c7de 100644
--- a/src/TorperfProcessor.java
+++ b/src/TorperfProcessor.java
@@ -11,6 +11,7 @@ public class TorperfProcessor {
     File torperfDir = new File(torperfDirectory);
     SortedMap<String, String> rawObs = new TreeMap<String, String>();
     SortedMap<String, String> stats = new TreeMap<String, String>();
+    int addedRawObs = 0;
     try {
       if (rawFile.exists()) {
         logger.fine("Reading file " + rawFile.getAbsolutePath() + "...");
@@ -84,6 +85,7 @@ public class TorperfProcessor {
                 String value = key + "," + completeMillis;
                 if (!rawObs.containsKey(key)) {
                   rawObs.put(key, value);
+                  addedRawObs++;
                   // TODO if torperf-stats generation takes long, compile
                   // list of dates that have changes for torperf-stats and
                   // only re-generate those
@@ -154,6 +156,30 @@ public class TorperfProcessor {
           + rawFile.getAbsolutePath() + " or "
           + statsFile.getAbsolutePath() + "!", e);
     }
+
+    /* Write stats. */
+    StringBuilder dumpStats = new StringBuilder("Finished writing "
+        + "statistics on torperf results.\nAdded " + addedRawObs
+        + " new observations in this execution.\n"
+        + "Last known obserations by source and file size are:");
+    String lastSource = null;
+    String lastLine = null;
+    for (String s : rawObs.keySet()) {
+      String[] parts = s.split(",");
+      if (lastSource == null) {
+        lastSource = parts[0];
+      } else if (!parts[0].equals(lastSource)) {
+        dumpStats.append("\n" + lastSource + " " + lastLine.split(" ")[1]
+            + " " + lastLine.split(" ")[2]);
+        lastSource = parts[0];
+      }
+      lastLine = s;
+    }
+    if (lastSource != null) {
+      dumpStats.append("\n" + lastSource + " " + lastLine.split(" ")[1]
+          + " " + lastLine.split(" ")[2]);
+    }
+    logger.info(dumpStats.toString());
   }
 }
 
-- 
1.6.5