[or-cvs] [metrics-utils/master 1/2] Warn if we're missing exit lists.

[karsten@xxxxxxxxxxxxxx]

Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Mon, 20 Sep 2010 11:27:23 +0200
Subject: Warn if we're missing exit lists.
Commit: 8c98b52a4a2eb35bf69750a8c5f5ba15b85dcb15

---
 visitor/VisiTor.java |   57 +++++++++++++++++++++++++++++++++++++++++--------
 1 files changed, 47 insertions(+), 10 deletions(-)

diff --git a/visitor/VisiTor.java b/visitor/VisiTor.java
index 2407e17..7f58164 100644
--- a/visitor/VisiTor.java
+++ b/visitor/VisiTor.java
@@ -74,6 +74,10 @@ public final class VisiTor {
     SimpleDateFormat exitListFilenameFormat = new SimpleDateFormat(
         "yyyy-MM-dd-HH-mm-ss");
     exitListFilenameFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    long firstExitList = -1L, lastExitList = -1L;
+    SimpleDateFormat isoDateTimeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm:ss");
+    isoDateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
     while (!fileStack.empty()) {
       File dirOrFile = fileStack.pop();
       if (dirOrFile.isDirectory()) {
@@ -82,7 +86,14 @@ public final class VisiTor {
         }
       } else {
         try {
-          exitListFilenameFormat.parse(dirOrFile.getName());
+          long timestamp = exitListFilenameFormat.parse(
+              dirOrFile.getName()).getTime();
+          if (firstExitList < 0 || timestamp < firstExitList) {
+            firstExitList = timestamp;
+          }
+          if (lastExitList < 0 || timestamp > lastExitList) {
+            lastExitList = timestamp;
+          }
           exitListFiles.add(dirOrFile);
         } catch (ParseException e) {
           /* Must be an unrelated file. Ingore it. */
@@ -95,9 +106,8 @@ public final class VisiTor {
       return;
     }
     System.out.println("OK\nWe found " + exitListFiles.size()
-        + " exit lists with the first list being "
-        + exitListFiles.first().getName() + " and the last list being "
-        + exitListFiles.last().getName() + ".");
+        + " exit lists between " + isoDateTimeFormat.format(firstExitList)
+        + " and " + isoDateTimeFormat.format(lastExitList) + ".");
 
     /* Read all ExitAddress lines from the provided exit list archives
      * and store them in a sorted set. */
@@ -142,9 +152,10 @@ public final class VisiTor {
       BufferedReader br = new BufferedReader(new FileReader(
           webServerLog));
       String line = null;
+      boolean haveWarnedAboutMissingExitListsStart = false,
+          haveWarnedAboutMissingExitListsEnd = false;
       while ((line = br.readLine()) != null) {
         String[] lineParts = line.split(" ");
-        String address = lineParts[0];
         long timestamp = -1L;
         try {
           timestamp = logFormat.parse(lineParts[3] + " " + lineParts[4]).
@@ -155,8 +166,29 @@ public final class VisiTor {
           e.printStackTrace();
           break;
         }
+        if (timestamp < firstExitList) {
+          if (!haveWarnedAboutMissingExitListsStart) {
+            System.out.print("WARN\nWe are missing exit lists before "
+                + isoDateTimeFormat.format(firstExitList) + " that we "
+                + "need  to parse the beginning of the server log file!\n"
+                + "Skipping... ");
+            haveWarnedAboutMissingExitListsStart = true;
+          }
+          continue;
+        }
+        if (timestamp > lastExitList) {
+          if (!haveWarnedAboutMissingExitListsEnd) {
+            System.out.print("WARN\nWe are missing exit lists after "
+                + isoDateTimeFormat.format(lastExitList) + " that we "
+                + "need to parse the end of the server log file!\n"
+                + "Skipping... ");
+            haveWarnedAboutMissingExitListsEnd = true;
+          }
+          continue;
+        }
         String currentDate = isoDateFormat.format(timestamp);
         allDates.add(currentDate);
+        String address = lineParts[0];
         String exitAddressLine = "ExitAddress " + address + " "
             + exitAddressFormat.format(timestamp);
         String previousExitAddressLine = exitAddressLines.headSet(
@@ -212,11 +244,16 @@ public final class VisiTor {
       bw.write("date,tor,nottor\n");
       String currentDate = allDates.first(), lastDate = allDates.last();
       while (currentDate.compareTo(lastDate) < 0) {
-        bw.write(currentDate + ","
-            + (torRequests.containsKey(currentDate)
-            ? torRequests.get(currentDate) : "NA") + ","
-            + (nonTorRequests.containsKey(currentDate)
-            ? nonTorRequests.get(currentDate) : "NA") + "\n");
+        if (!torRequests.containsKey(currentDate) &&
+            !nonTorRequests.containsKey(currentDate)) {
+          bw.write(currentDate + ",NA,NA\n");
+        } else {
+          bw.write(currentDate + ","
+              + (torRequests.containsKey(currentDate)
+              ? torRequests.get(currentDate) : "0") + ","
+              + (nonTorRequests.containsKey(currentDate)
+              ? nonTorRequests.get(currentDate) : "0") + "\n");
+        }
         try {
           currentDate = isoDateFormat.format(isoDateFormat.parse(
               currentDate).getTime() + 24L * 60L * 60L * 1000L);
-- 
1.7.1