[or-cvs] [ernie/master 2/2] Keep history of imported relay descriptors.

[karsten@xxxxxxxxxxxxxx]

Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Mon, 29 Mar 2010 12:30:40 +0200
Subject: Keep history of imported relay descriptors.
Commit: 3b4a9cd679a6ca707bc7c3b829a92694f47122e0

---
 config                 |    6 ++++++
 src/ArchiveReader.java |   42 ++++++++++++++++++++++++++++++++++++++++--
 src/Configuration.java |    7 +++++++
 src/Main.java          |    3 ++-
 4 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/config b/config
index caf304f..e208ffc 100644
--- a/config
+++ b/config
@@ -55,6 +55,12 @@
 ## Import directory archives from disk, if available
 #ImportDirectoryArchives 1
 
+## Keep a history of imported directory archive files to know which files
+## have been imported before. This history can be useful when importing
+## from a changing source to avoid importing descriptors over and over
+## again, but it can be confusing to users who don't know about it.
+#KeepDirectoryArchiveImportHistory 0
+
 ## Write relay descriptors to a database for later evaluation
 #WriteRelayDescriptorDatabase 0
 
diff --git a/src/ArchiveReader.java b/src/ArchiveReader.java
index cebf919..7c55c41 100644
--- a/src/ArchiveReader.java
+++ b/src/ArchiveReader.java
@@ -8,8 +8,26 @@ import org.apache.commons.compress.compressors.bzip2.*;
  * them to the relay descriptor parser.
  */
 public class ArchiveReader {
-  public ArchiveReader(RelayDescriptorParser rdp, String archivesDir) {
+  public ArchiveReader(RelayDescriptorParser rdp, String archivesDir,
+      boolean keepImportHistory) {
     Logger logger = Logger.getLogger(ArchiveReader.class.getName());
+    SortedSet<String> archivesImportHistory = new TreeSet<String>();
+    File archivesImportHistoryFile =
+        new File("stats/archives-import-history");
+    if (keepImportHistory && archivesImportHistoryFile.exists()) {
+      try {
+        BufferedReader br = new BufferedReader(new FileReader(
+            archivesImportHistoryFile));
+        String line = null;
+        while ((line = br.readLine()) != null) {
+          archivesImportHistory.add(line);
+        }
+        br.close();
+      } catch (IOException e) {
+        logger.log(Level.WARNING, "Could not read in archives import "
+            + "history file. Skipping.");
+      }
+    }
     if (new File(archivesDir).exists()) {
       logger.fine("Importing files in directory " + archivesDir
           + "/...");
@@ -27,7 +45,10 @@ public class ArchiveReader {
             try {
               BufferedInputStream bis = null;
               System.out.println(pop.getName());
-              if (pop.getName().endsWith("\\.tar\\.bz2")) {
+              if (keepImportHistory &&
+                  archivesImportHistory.contains(pop.getName())) {
+                continue;
+              } else if (pop.getName().endsWith(".tar.bz2")) {
                 logger.warning("Cannot parse compressed tarball "
                     + pop.getAbsolutePath() + ". Skipping.");
                 continue;
@@ -40,6 +61,9 @@ public class ArchiveReader {
                 FileInputStream fis = new FileInputStream(pop);
                 bis = new BufferedInputStream(fis);
               }
+              if (keepImportHistory) {
+                archivesImportHistory.add(pop.getName());
+              }
               ByteArrayOutputStream baos = new ByteArrayOutputStream();
               int len;
               byte[] data = new byte[1024];
@@ -74,6 +98,20 @@ public class ArchiveReader {
         }
       }
     }
+    if (keepImportHistory) {
+      try {
+        archivesImportHistoryFile.getParentFile().mkdirs();
+        BufferedWriter bw = new BufferedWriter(new FileWriter(
+            archivesImportHistoryFile));
+        for (String line : archivesImportHistory) {
+          bw.write(line + "\n");
+        }
+        bw.close();
+      } catch (IOException e) {
+        logger.log(Level.WARNING, "Could not write archives import "
+            + "history file.");
+      }
+    }
   }
 }
 
diff --git a/src/Configuration.java b/src/Configuration.java
index 6f6a854..7c27139 100644
--- a/src/Configuration.java
+++ b/src/Configuration.java
@@ -25,6 +25,7 @@ public class Configuration {
   private boolean writeDirectoryArchives = false;
   private boolean importCachedRelayDescriptors = true;
   private boolean importDirectoryArchives = true;
+  private boolean keepDirectoryArchiveImportHistory = false;
   private boolean writeRelayDescriptorDatabase = false;
   private String relayDescriptorDatabaseJdbc = "jdbc:postgresql:tordir?"
         + "user=ernie&password=password";
@@ -93,6 +94,9 @@ public class Configuration {
         } else if (line.startsWith("ImportDirectoryArchives")) {
           this.importDirectoryArchives = Integer.parseInt(
               line.split(" ")[1]) != 0;
+        } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
+          this.keepDirectoryArchiveImportHistory = Integer.parseInt(
+              line.split(" ")[1]) != 0;
         } else if (line.startsWith("WriteRelayDescriptorDatabase")) {
           this.writeRelayDescriptorDatabase = Integer.parseInt(
               line.split(" ")[1]) != 0;
@@ -203,6 +207,9 @@ public class Configuration {
   public boolean getImportDirectoryArchives() {
     return this.importDirectoryArchives;
   }
+  public boolean getKeepDirectoryArchiveImportHistory() {
+    return this.keepDirectoryArchiveImportHistory;
+  }
   public boolean getWriteRelayDescriptorDatabase() {
     return this.writeRelayDescriptorDatabase;
   }
diff --git a/src/Main.java b/src/Main.java
index 53a4187..8a2ee06 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -81,7 +81,8 @@ public class Main {
         new CachedRelayDescriptorReader(rdp);
       }
       if (config.getImportDirectoryArchives()) {
-        new ArchiveReader(rdp, "archives");
+        new ArchiveReader(rdp, "archives",
+            config.getKeepDirectoryArchiveImportHistory());
       }
       if (rdd != null) {
         rdd.downloadMissingDescriptors();
-- 
1.6.5