[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] [ernie/master] Maintain a list of descriptors we want to download.



Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed, 24 Feb 2010 17:47:37 +0100
Subject: Maintain a list of descriptors we want to download.
Commit: 7503c5db66ab6d372c4e19ce55740e40f6f701a4

---
 config                               |    3 +
 src/ArchiveWriter.java               |  487 +++++++++++++++++++++++++++++-----
 src/CachedRelayDescriptorReader.java |   63 +++---
 src/Configuration.java               |   14 +
 src/Main.java                        |   15 +-
 src/RelayDescriptorDownloader.java   |  137 ++++++-----
 src/RelayDescriptorParser.java       |   24 ++-
 7 files changed, 569 insertions(+), 174 deletions(-)

diff --git a/config b/config
index 37fea66..61c24c7 100644
--- a/config
+++ b/config
@@ -39,6 +39,9 @@
 ## Write directory archives to disk
 #WriteDirectoryArchives 1
 
+## V3 directory authority fingerprints
+#V3DirectoryAuthorities 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,D586D18309DED4CD6D57C18FDB97EFA96D330566,585769C78764D58426B8B52B6651A5A71137189A,27B6B5996C426270A5C95488AA5BCEB6BCC86956,80550987E1D626E3EBA5E5E75A458DE0626D088C,ED03BB616EB2F60BEC80151114BB25CEF515B226
+
 ## Read cached-* files from a local Tor client
 #ImportCachedRelayDescriptors 1
 
diff --git a/src/ArchiveWriter.java b/src/ArchiveWriter.java
index 2ebe423..73c8030 100644
--- a/src/ArchiveWriter.java
+++ b/src/ArchiveWriter.java
@@ -1,88 +1,449 @@
 import java.io.*;
 import java.text.*;
 import java.util.*;
-import org.apache.commons.compress.archivers.tar.*;
+import java.util.logging.*;
+import org.apache.commons.codec.digest.*;
+import org.apache.commons.codec.binary.*;
 
 public class ArchiveWriter {
-  public ArchiveWriter() {
-  }
-  public void storeConsensus(BufferedReader br, String validAfterTime)
-      throws IOException, ParseException {
+  private String statsDir;
+  private SortedSet<String> v3DirectoryAuthorities;
+  private File archiveWriterParseHistory;
+  private SortedSet<String> missingDescriptors;
+  private String lastParsedConsensus;
+  private boolean initialized = false;
+  private boolean archiveWriterParseHistoryModified = false;
+  private Logger logger;
+  private String parseTime;
+  public ArchiveWriter(String statsDir,
+      SortedSet<String> v3DirectoryAuthorities) {
+    this.statsDir = statsDir;
+    this.v3DirectoryAuthorities = v3DirectoryAuthorities;
+    this.archiveWriterParseHistory = new File(statsDir
+        + "/archive-writer-parse-history");
+    this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
     SimpleDateFormat parseFormat =
         new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
     parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    long validAfter = parseFormat.parse(validAfterTime).getTime();
-    SimpleDateFormat printFormat =
+    this.parseTime = parseFormat.format(new Date());
+    this.missingDescriptors = new TreeSet<String>();
+    SimpleDateFormat consensusVoteFormat =
         new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
-    printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    File consensusFile = new File("directory-archive/consensus/"
-        + printFormat.format(new Date(validAfter)) + "-consensus");
-    consensusFile.getParentFile().mkdirs();
-    if (!consensusFile.exists()) {
-      BufferedWriter bw = new BufferedWriter(new FileWriter(
-          consensusFile));
-      String line = null;
-      while ((line = br.readLine()) != null) {
-        bw.write(line + "\n");
-        if (line.startsWith("r ")) {
-          // TODO compile list of server descriptors that we might want to
-          // learn about
+    consensusVoteFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    SimpleDateFormat descriptorFormat =
+        new SimpleDateFormat("yyyy/MM/");
+    descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    if (this.archiveWriterParseHistory.exists()) {
+      this.logger.info("Reading file " + statsDir
+          + "/archive-writer-parse-history...");
+      try {
+        BufferedReader br = new BufferedReader(new FileReader(
+            this.archiveWriterParseHistory));
+        String line = null;
+        SimpleDateFormat publishedFormat =
+            new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+        publishedFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+        long now = System.currentTimeMillis();
+        while ((line = br.readLine()) != null) {
+          // only add to download list if descriptors are still available
+          // on directories
+          long published = publishedFormat.parse(line.split(",")[2]).
+              getTime();
+          if (line.startsWith("consensus") &&
+              published + 55L * 60L * 1000L > now &&
+              !new File("directory-archive/consensus/"
+                + consensusVoteFormat.format(new Date(published))
+                + "-consensus").exists()) {
+            this.logger.fine("Initializing missing list with "
+                + "consensus: valid-after=" + line.split(",")[2]
+                + ", filename=directory-archive/consensus/"
+                + consensusVoteFormat.format(new Date(published))
+                + "-consensus");
+            this.missingDescriptors.add(line);
+          } else if (line.startsWith("vote") &&
+              published + 55L * 60L * 1000L > now &&
+              // TODO is vote even available for 55 minutes after its
+              // publication?
+              !new File("directory-archive/vote/"
+                + consensusVoteFormat.format(new Date(published))
+                + "-vote-" + line.split(",")[1]).exists()) {
+            this.logger.fine("Initializing missing list with vote: "
+                + "fingerprint=" + line.split(",")[1]
+                + ", valid-after="
+                + consensusVoteFormat.format(new Date(published))
+                + ", filename=directory-archive/vote/"
+                + consensusVoteFormat.format(new Date(published))
+                + "-vote-" + line.split(",")[1]);
+            this.missingDescriptors.add(line);
+          } else if ((line.startsWith("server") ||
+              line.startsWith("extra")) &&
+              published + 24L * 60L * 60L * 1000L > now) {
+              // TODO are 24 hours okay?
+            boolean isServerDesc = line.startsWith("server");
+            String digest = line.split(",")[1].toLowerCase();
+            if (!new File("directory-archive/"
+                + (isServerDesc ? "server-descriptor" : "extra-info")
+                + "/" + descriptorFormat.format(new Date(published))
+                + digest.substring(0, 1) + "/" + digest.substring(1, 2)
+                + "/" + digest).exists()) {
+              this.logger.fine("Initializing missing list with "
+                  + (isServerDesc ? "server" : "extra-info")
+                  + " descriptor: digest=" + digest
+                  + ", filename=directory-archive/server-descriptor/"
+                  + descriptorFormat.format(new Date(published))
+                  + line.split(",")[1].substring(0, 1) + "/"
+                  + line.split(",")[1].substring(1, 2) + "/"
+                  + line.split(",")[1]);
+              this.missingDescriptors.add(line);
+            }
+          }
         }
+        br.close();
+        this.logger.info("Finished reading file " + statsDir
+            + "/archive-writer-parse-history");
+      } catch (ParseException e) {
+        this.logger.log(Level.WARNING, "Failed reading file " + statsDir
+            + "/archive-writer-parse-history! This means that we might "
+            + "forget to dowload descriptors we are missing.", e);
+      } catch (IOException e) {
+        this.logger.log(Level.WARNING, "Failed reading file " + statsDir
+            + "/archive-writer-parse-history! This means that we might "
+            + "forget to dowload descriptors we are missing.", e);
+      }
+    }
+    // add current consensus and votes to list
+    SimpleDateFormat consensusFormat =
+        new SimpleDateFormat("yyyy-MM-dd HH");
+    consensusFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    String nowConsensusFormat = consensusFormat.format(new Date())
+        + ":00:00";
+    long nowConsensus = (System.currentTimeMillis() / (60L * 60L * 1000L))
+        * (60L * 60L * 1000L);
+    for (String authority : this.v3DirectoryAuthorities) {
+      if (!new File("directory-archive/vote/"
+            + consensusVoteFormat.format(new Date(nowConsensus))
+            + "-vote-" + authority).exists()) {
+        if (!this.missingDescriptors.contains("vote," + authority + ","
+            + nowConsensusFormat)) {
+          this.logger.fine("Adding vote to missing list: fingerprint="
+              + authority + ", valid-after="
+              + consensusVoteFormat.format(new Date(nowConsensus))
+              + ", filename=directory-archive/vote/"
+              + consensusVoteFormat.format(new Date(nowConsensus))
+              + "-vote-" + authority);
+          this.missingDescriptors.add("vote," + authority + ","
+              + nowConsensusFormat);
+          this.archiveWriterParseHistoryModified = true;
+        }
+      }
+    }
+    if (!new File("directory-archive/consensus/"
+        + consensusVoteFormat.format(new Date(nowConsensus))
+        + "-consensus").exists()) {
+      if (!this.missingDescriptors.contains("consensus,NA,"
+          + nowConsensusFormat)) {
+        this.logger.fine("Adding consensus to missing list: valid-after="
+            + nowConsensusFormat
+            + ", filename=directory-archive/consensus/"
+            + consensusVoteFormat.format(new Date(nowConsensus))
+            + "-consensus");
+        this.missingDescriptors.add("consensus,NA,"
+            + nowConsensusFormat);
+        this.archiveWriterParseHistoryModified = true;
       }
-      bw.close();
     }
   }
-  public void storeVote(BufferedReader br, String validAfterTime,
-      String authorityFingerprint) throws IOException {
-    // TODO implement me
-  }
-  public void storeServerDescriptor(BufferedReader br, String digest,
-      String publishedTime, String extraInfoDigest) throws IOException,
+  public void store(BufferedReader br) throws IOException,
       ParseException {
+    String line = br.readLine();
+    if (line == null) {
+      this.logger.warning("Someone gave us an empty file for storing!");
+      return;
+    }
+    StringBuilder sb = new StringBuilder();
+    sb.append(line + "\n");
     SimpleDateFormat parseFormat =
         new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
     parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    long published = parseFormat.parse(publishedTime).getTime();
-    SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
-    printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    File descriptorFile = new File("directory-archive/server-descriptor/"
-        + printFormat.format(new Date(published))
-        + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
-        + digest);
-    descriptorFile.getParentFile().mkdirs();
-    if (!descriptorFile.exists()) {
-      BufferedWriter bw = new BufferedWriter(new FileWriter(
-          descriptorFile));
-      String line = null;
+    if (line.equals("network-status-version 3")) {
+      // TODO when parsing the current consensus, check the fresh-until
+      // time to see when we switch from hourly to half-hourly
+      // consensuses; in that case, add next half-hourly consensus to
+      // missing list and warn!
+      boolean isConsensus = true;
+      String validAfterTime = null;
+      long validAfter = -1L;
+      long now = System.currentTimeMillis();
+      String fingerprint = null;
+      SimpleDateFormat descriptorFormat =
+          new SimpleDateFormat("yyyy/MM/");
+      descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
       while ((line = br.readLine()) != null) {
-        bw.write(line + "\n");
+        sb.append(line + "\n");
+        if (line.equals("vote-status vote")) {
+          isConsensus = false;
+        } else if (line.startsWith("valid-after ")) {
+          validAfterTime = line.substring("valid-after ".length());
+          validAfter = parseFormat.parse(validAfterTime).getTime();
+        } else if (line.startsWith("dir-source ") &&
+            !this.v3DirectoryAuthorities.contains(
+            line.split(" ")[2]) && validAfter + 55L * 60L * 1000L <
+            System.currentTimeMillis()) {
+          this.logger.warning("Unknown v3 directory authority fingerprint "
+              + "in consensus line '" + line + "'. You should update your "
+              + "V3DirectoryAuthorities config option!");
+          fingerprint = line.split(" ")[2];
+          long nowConsensus = (now / (60L * 60L * 1000L))
+              * (60L * 60L * 1000L);
+          SimpleDateFormat consensusVoteFormat =
+              new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+          consensusVoteFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+          if (!new File("directory-archive/vote/"
+                + consensusVoteFormat.format(new Date(nowConsensus))
+                + "-vote-" + fingerprint).exists()) {
+            if (!this.missingDescriptors.contains("vote," + fingerprint
+                + "," + parseFormat.format(new Date(nowConsensus)))) {
+              this.logger.fine("Adding vote to missing list: fingerprint="
+                  + fingerprint + ", valid-after="
+                  + parseFormat.format(new Date(nowConsensus))
+                  + ", filename=directory-archive/vote/"
+                  + consensusVoteFormat.format(new Date(nowConsensus))
+                  + "-vote-" + fingerprint);
+              this.missingDescriptors.add("vote," + fingerprint + ","
+                  + parseFormat.format(new Date(nowConsensus)));
+              this.archiveWriterParseHistoryModified = true;
+            }
+          }
+        } else if (line.startsWith("fingerprint ")) {
+          fingerprint = line.split(" ")[1];
+        } else if (line.startsWith("r ")) {
+          String publishedTime = line.split(" ")[4] + " "
+              + line.split(" ")[5];
+          long published = parseFormat.parse(publishedTime).getTime();
+          String digest = Hex.encodeHexString(Base64.decodeBase64(
+              line.split(" ")[3] + "=")).toLowerCase();
+          // TODO are 24 hours okay?
+          if (published + 24L * 60L * 60L * 1000L > now &&
+              !new File("directory-archive/server-descriptor/"
+              + descriptorFormat.format(new Date(published))
+              + digest.substring(0, 1) + "/" + digest.substring(1, 2)
+              + "/" + digest).exists()) {
+            if (!this.missingDescriptors.contains("server," + digest + ","
+                + publishedTime)) {
+              this.logger.fine("Adding server descriptor to missing list: "
+                  + "digest=" + digest
+                  + ", filename=directory-archive/server-descriptor/"
+                  + descriptorFormat.format(new Date(published))
+                  + digest.substring(0, 1) + "/" + digest.substring(1, 2)
+                  + "/" + digest);
+              this.missingDescriptors.add("server," + digest + ","
+                  + publishedTime);
+              this.archiveWriterParseHistoryModified = true;
+            }
+          }
+        }
+      }
+      SimpleDateFormat printFormat =
+          new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+      printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+      if (isConsensus) {
+        File consensusFile = new File("directory-archive/consensus/"
+            + printFormat.format(new Date(validAfter)) + "-consensus");
+        if (!consensusFile.exists()) {
+          this.logger.fine("Storing consensus: valid-after="
+              + validAfterTime + ", filename=directory-archive/consensus/"
+              + printFormat.format(new Date(validAfter)) + "-consensus");
+          consensusFile.getParentFile().mkdirs();
+          BufferedReader br2 = new BufferedReader(new StringReader(
+              sb.toString()));
+          BufferedWriter bw = new BufferedWriter(new FileWriter(
+              consensusFile));
+          while ((line = br2.readLine()) != null) {
+              bw.write(line + "\n");
+          }
+          bw.close();
+          br2.close();
+          this.logger.fine("Removing consensus from missing list: "
+              + "valid-after=" + validAfterTime
+              + ", filename=directory-archive/consensus/"
+              + printFormat.format(new Date(validAfter)) + "-consensus");
+          this.missingDescriptors.remove("consensus,NA,"
+              + validAfterTime);
+          this.archiveWriterParseHistoryModified = true;
+        } else {
+          this.logger.info("Not storing consensus, because we already "
+              + "have it: valid-after=" + validAfterTime
+              + ", filename=directory-archive/consensus/"
+              + printFormat.format(new Date(validAfter)) + "-consensus");
+        }
+      } else {
+        File voteFile = new File("directory-archive/vote/"
+            + printFormat.format(new Date(validAfter)) + "-vote-"
+            + fingerprint);
+        if (!voteFile.exists()) {
+          this.logger.fine("Storing vote: fingerprint=" + fingerprint
+              + ", valid-after="
+              + printFormat.format(new Date(validAfter))
+              + ", filename=directory-archive/vote/"
+              + printFormat.format(new Date(validAfter)) + "-vote-"
+              + fingerprint);
+          voteFile.getParentFile().mkdirs();
+          BufferedReader br2 = new BufferedReader(new StringReader(
+              sb.toString()));
+          BufferedWriter bw = new BufferedWriter(new FileWriter(
+              voteFile));
+          while ((line = br2.readLine()) != null) {
+              bw.write(line + "\n");
+          }
+          bw.close();
+          br2.close();
+          this.logger.fine("Removing vote from missing list: "
+              + "fingerprint=" + fingerprint + ", valid-after="
+              + printFormat.format(new Date(validAfter))
+              + ", filename=directory-archive/vote/"
+              + printFormat.format(new Date(validAfter)) + "-vote-"
+              + fingerprint);
+          this.missingDescriptors.remove("vote," + fingerprint + ","
+              + validAfterTime);
+          this.archiveWriterParseHistoryModified = true;
+        } else {
+          this.logger.info("Not storing vote, because we already have "
+              + "it: fingerprint=" + fingerprint + ", valid-after="
+              + printFormat.format(new Date(validAfter))
+              + ", filename=directory-archive/vote/"
+              + printFormat.format(new Date(validAfter)) + "-vote-"
+              + fingerprint);
+        }
+      }
+    } else if (line.startsWith("router ") ||
+        line.startsWith("extra-info ")) {
+      boolean isServerDescriptor = line.startsWith("router ");
+      String publishedTime = null;
+      long published = -1L;
+      String digest = null;
+      while ((line = br.readLine()) != null) {
+        sb.append(line + "\n");
+        if (line.startsWith("published ")) {
+          publishedTime = line.substring("published ".length());
+          published = parseFormat.parse(publishedTime).getTime();
+        } else if (line.startsWith("opt extra-info-digest ") ||
+            line.startsWith("extra-info-digest ")) {
+          String extraInfoDigest = line.startsWith("opt ") ?
+              line.split(" ")[2].toLowerCase() :
+              line.split(" ")[1].toLowerCase();
+          SimpleDateFormat descriptorFormat =
+              new SimpleDateFormat("yyyy/MM/");
+          descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+          if (!new File("directory-archive/extra-info/"
+              + descriptorFormat.format(new Date(published))
+              + extraInfoDigest.substring(0, 1) + "/"
+              + extraInfoDigest.substring(1, 2) + "/"
+              + extraInfoDigest).exists()) {
+            if (!this.missingDescriptors.contains("extra,"
+                + extraInfoDigest + "," + publishedTime)) {
+              this.logger.fine("Adding extra-info descriptor to missing "
+                  + "list: digest=" + extraInfoDigest
+                  + ", filename=directory-archive/extra-info/"
+                  + descriptorFormat.format(new Date(published))
+                  + extraInfoDigest.substring(0, 1) + "/"
+                  + extraInfoDigest.substring(1, 2) + "/"
+                  + extraInfoDigest);
+              this.missingDescriptors.add("extra," + extraInfoDigest + ","
+                  + publishedTime);
+              this.archiveWriterParseHistoryModified = true;
+            }
+          }
+        } else if (line.equals("router-signature")) {
+          digest = DigestUtils.shaHex(sb.toString()).toLowerCase();
+        }
+      }
+      SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
+      printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+      File descriptorFile = new File("directory-archive/"
+          + (isServerDescriptor ? "server-descriptor" : "extra-info") + "/"
+          + printFormat.format(new Date(published))
+          + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
+          + digest);
+      if (!descriptorFile.exists()) {
+        this.logger.fine("Storing " + (isServerDescriptor ?
+            "server descriptor" : "extra-info descriptor")
+            + ": digest=" + digest + ", filename=directory-archive/"
+            + (isServerDescriptor ? "server-descriptor" : "extra-info")
+            + "/" + printFormat.format(new Date(published))
+            + digest.substring(0, 1) + "/" + digest.substring(1, 2)
+            + "/" + digest);
+        descriptorFile.getParentFile().mkdirs();
+        BufferedReader br2 = new BufferedReader(new StringReader(
+            sb.toString()));
+        BufferedWriter bw = new BufferedWriter(new FileWriter(
+            descriptorFile));
+        while ((line = br2.readLine()) != null) {
+          bw.write(line + "\n");
+        }
+        bw.close();
+        br2.close();
+        this.logger.fine("Removing " + (isServerDescriptor ?
+            "server descriptor" : "extra-info descriptor")
+            + " from missing list: digest=" + digest
+            + ", filename=directory-archive/"
+            + (isServerDescriptor ? "server-descriptor" : "extra-info")
+            + "/" + printFormat.format(new Date(published))
+            + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
+            + digest);
+        if (isServerDescriptor) {
+          this.missingDescriptors.remove("server," + digest + ","
+              + publishedTime);
+        } else {
+          this.missingDescriptors.remove("extra," + digest + ","
+              + publishedTime);
+        }
+        this.archiveWriterParseHistoryModified = true;
+      } else {
+        this.logger.info("Not storing " + (isServerDescriptor ?
+            "server descriptor" : "extra-info descriptor")
+            + ", because we already have it: digest=" + digest
+            + ", filename=directory-archive/"
+            + (isServerDescriptor ? "server-descriptor" : "extra-info")
+            + "/" + printFormat.format(new Date(published))
+            + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
+            + digest);
       }
-      bw.close();
     }
-    // TODO if extraInfoDigest != null, add digest to extra-info
-    // descriptors we want to download
   }
-  public void storeExtraInfo(BufferedReader br, String digest,
-      String publishedTime) throws IOException, ParseException {
-    SimpleDateFormat parseFormat =
-        new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-    parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    long published = parseFormat.parse(publishedTime).getTime();
-    SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
-    printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    File descriptorFile = new File("directory-archive/extra-info/"
-        + printFormat.format(new Date(published))
-        + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
-        + digest);
-    descriptorFile.getParentFile().mkdirs();
-    if (!descriptorFile.exists()) {
-      BufferedWriter bw = new BufferedWriter(new FileWriter(
-          descriptorFile));
-      String line = null, extraInfoDigest = null;
-      while ((line = br.readLine()) != null) {
-        bw.write(line + "\n");
+  public Set<String> getMissingDescriptorUrls() {
+    Set<String> urls = new HashSet<String>();
+    for (String line : this.missingDescriptors) {
+      if (line.startsWith("consensus,")) {
+        urls.add("/tor/status-vote/current/consensus");
+      } else if (line.startsWith("vote,")) {
+        urls.add("/tor/status-vote/current/" + line.split(",")[1]);
+      } else if (line.startsWith("server,")) {
+        urls.add("/tor/server/d/" + line.split(",")[1]);
+      } else if (line.startsWith("extra,")) {
+        urls.add("/tor/extra/d/" + line.split(",")[1]);
+      }
+    }
+    return urls;
+  }
+  public void writeFile() {
+    if (this.archiveWriterParseHistoryModified) {
+      try {
+        this.logger.info("Writing file " + this.statsDir
+            + "/archive-writer-parse-history...");
+        new File(this.statsDir).mkdirs();
+        BufferedWriter bw = new BufferedWriter(new FileWriter(
+            this.archiveWriterParseHistory));
+        bw.write("type,source,published\n");
+        for (String line : this.missingDescriptors) {
+          bw.write(line + "\n");
+        }
+        bw.close();
+        this.logger.info("Finished writing file " + this.statsDir
+            + "/archive-writer-parse-history.");
+      } catch (IOException e) {
+        this.logger.log(Level.WARNING, "Failed writing " + this.statsDir
+            + "/archive-writer-parse-history!", e);
       }
-      bw.close();
     }
   }
 }
diff --git a/src/CachedRelayDescriptorReader.java b/src/CachedRelayDescriptorReader.java
index 8a7ed5f..d865b5c 100644
--- a/src/CachedRelayDescriptorReader.java
+++ b/src/CachedRelayDescriptorReader.java
@@ -1,5 +1,6 @@
 import java.io.*;
 import java.text.*;
+import java.util.logging.*;
 import org.apache.commons.codec.digest.*;
 
 /**
@@ -9,21 +10,33 @@ import org.apache.commons.codec.digest.*;
 public class CachedRelayDescriptorReader {
   public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
       ArchiveWriter aw) {
+    Logger logger = Logger.getLogger(
+        CachedRelayDescriptorReader.class.getName());
     File cachedDescDir = new File("cacheddesc");
     if (cachedDescDir.exists()) {
+      logger.info("Reading cacheddesc/ directory.");
       try {
-        rdp.initialize();
+        rdp.initialize(); // TODO get rid of this non-sense
       } catch (IOException e) {
         return;
       }
       for (File f : cachedDescDir.listFiles()) {
         try {
-          if (f.getName().equals("cached-consensus") ||
-              f.getName().startsWith("cached-descriptors") ||
+          if (f.getName().equals("cached-consensus")) {
+            BufferedReader br = new BufferedReader(new FileReader(f));
+            if (aw != null) {
+              aw.store(br);
+            }
+            br.close();
+            br = new BufferedReader(new FileReader(f));
+            if (rdp != null) {
+              rdp.parse(br);
+            }
+            br.close();
+          } else if (f.getName().startsWith("cached-descriptors") ||
               f.getName().startsWith("cached-extrainfo")) {
             BufferedReader br = new BufferedReader(new FileReader(f));
-            String line = null, validAfterTime = null, publishedTime = null,
-                extraInfoDigest = null, digest = null;
+            String line = null;
             StringBuilder sb = new StringBuilder();
             while ((line = br.readLine()) != null || sb != null) {
               if (line == null && sb.length() < 1) {
@@ -34,19 +47,14 @@ public class CachedRelayDescriptorReader {
                 if (sb.length() > 0) {
                   BufferedReader storeBr = new BufferedReader(
                       new StringReader(sb.toString()));
-                  if (f.getName().equals("cached-consensus")) {
-                    aw.storeConsensus(storeBr, validAfterTime);
-                    validAfterTime = null;
-                  } else if (f.getName().startsWith("cached-descriptors")) {
-                    aw.storeServerDescriptor(storeBr, digest,
-                        publishedTime, extraInfoDigest);
-                    digest = null;
-                    publishedTime = null;
-                    extraInfoDigest = null;
-                  } else if (f.getName().startsWith("cached-extrainfo")) {
-                    aw.storeExtraInfo(storeBr, digest, publishedTime);
-                    digest = null;
-                    publishedTime = null;
+                  if (aw != null) {
+                    aw.store(storeBr);
+                  }
+                  storeBr.close();
+                  storeBr = new BufferedReader(
+                      new StringReader(sb.toString()));
+                  if (rdp != null) {
+                    rdp.parse(storeBr);
                   }
                   storeBr.close();
                 }
@@ -57,28 +65,19 @@ public class CachedRelayDescriptorReader {
                   sb = new StringBuilder();
                 }
               }
-              if (line.startsWith("valid-after ")) {
-                validAfterTime = line.substring("valid-after ".length());
-              } else if (line.startsWith("published ")) {
-                publishedTime = line.substring("published ".length());
-              } else if (line.startsWith("router-signature")) {
-                digest = DigestUtils.shaHex(sb.toString()
-                    + "router-signature\n").toUpperCase();
-              } else if (line.startsWith("opt extra-info-digest ")) {
-                extraInfoDigest = line.split(" ")[2];
-              } else if (line.startsWith("extra-info-digest ")) {
-                extraInfoDigest = line.split(" ")[1];
-              }
               if (!line.startsWith("@")) {
                 sb.append(line + "\n");
               }
             }
             br.close();
+            logger.info("Finished reading cacheddesc/ directory.");
           }
         } catch (IOException e) {
-          // TODO handle
+          logger.log(Level.WARNING, "Failed reading cacheddesc/ "
+              + "directory.", e);
         } catch (ParseException e) {
-          // TODO handle
+          logger.log(Level.WARNING, "Failed reading cacheddesc/ "
+              + "directory.", e);
         }
       }
     }
diff --git a/src/Configuration.java b/src/Configuration.java
index 9eb8f3c..0eeb524 100644
--- a/src/Configuration.java
+++ b/src/Configuration.java
@@ -18,6 +18,14 @@ public class Configuration {
       + "9695DFC35FFEB861329B9F1AB04C46397020CE31").split(",")));
   private boolean writeBridgeStats = true;
   private boolean writeDirectoryArchives = true;
+  private SortedSet<String> v3DirectoryAuthorities = new TreeSet<String>(
+      Arrays.asList(("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,"
+      + "E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,"
+      + "D586D18309DED4CD6D57C18FDB97EFA96D330566,"
+      + "585769C78764D58426B8B52B6651A5A71137189A,"
+      + "27B6B5996C426270A5C95488AA5BCEB6BCC86956,"
+      + "80550987E1D626E3EBA5E5E75A458DE0626D088C,"
+      + "ED03BB616EB2F60BEC80151114BB25CEF515B226").split(",")));
   private boolean importCachedRelayDescriptors = true;
   private boolean importDirectoryArchives = true;
   private boolean importSanitizedBridges = true;
@@ -67,6 +75,9 @@ public class Configuration {
         } else if (line.startsWith("WriteDirectoryArchives")) {
           this.writeDirectoryArchives = Integer.parseInt(
               line.split(" ")[1]) != 0;
+        } else if (line.startsWith("V3DirectoryAuthorities")) {
+          this.v3DirectoryAuthorities = new TreeSet<String>(
+              Arrays.asList(line.split(" ")[1].split(",")));
         } else if (line.startsWith("ImportCachedRelayDescriptors")) {
           this.importCachedRelayDescriptors = Integer.parseInt(
               line.split(" ")[1]) != 0;
@@ -155,6 +166,9 @@ public class Configuration {
   public boolean getWriteDirectoryArchives() {
     return this.writeDirectoryArchives;
   }
+  public SortedSet<String> getV3DirectoryAuthorities() {
+    return this.v3DirectoryAuthorities;
+  }
   public boolean getImportCachedRelayDescriptors() {
     return this.importCachedRelayDescriptors;
   }
diff --git a/src/Main.java b/src/Main.java
index c52313c..8d8b2a1 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -47,7 +47,8 @@ public class Main {
 
     // Prepare writing relay descriptor archive to disk
     ArchiveWriter aw = config.getWriteDirectoryArchives() ?
-        new ArchiveWriter() : null;
+        new ArchiveWriter(statsDirectory,
+        config.getV3DirectoryAuthorities()) : null;
     // TODO handle case aw==NULL below
 
     // import and/or download relay and bridge descriptors
@@ -58,14 +59,9 @@ public class Main {
       new ArchiveReader(rdp, "archives");
     }
     if (config.getDownloadRelayDescriptors()) {
-      // TODO make this smarter by letting rdd ask rdp which descriptors
-      // are still missing and only download those
-      // TODO move iteration over dirauths from main() to class code
-      for (String directoryAuthority : 
-          config.getDownloadFromDirectoryAuthorities()) {
-        new RelayDescriptorDownloader(rdp, aw, directoryAuthority,
-            directories);
-      }
+      new RelayDescriptorDownloader(rdp, aw,
+          config.getDownloadFromDirectoryAuthorities(),
+          directories);
     }
     if (config.getImportSanitizedBridges()) {
       new SanitizedBridgesReader(bdp, "bridges", countries);
@@ -86,6 +82,7 @@ public class Main {
       dsfh.writeFile();
     }
     rdp.writeFile();
+    aw.writeFile();
 
     // Import and process torperf stats
     if (config.getImportWriteTorperfStats()) {
diff --git a/src/RelayDescriptorDownloader.java b/src/RelayDescriptorDownloader.java
index 7f953be..b0f270d 100644
--- a/src/RelayDescriptorDownloader.java
+++ b/src/RelayDescriptorDownloader.java
@@ -1,5 +1,6 @@
 import java.io.*;
 import java.net.*;
+import java.text.*;
 import java.util.*;
 import java.util.logging.*;
 
@@ -9,74 +10,86 @@ import java.util.logging.*;
  */
 public class RelayDescriptorDownloader {
   public RelayDescriptorDownloader(RelayDescriptorParser rdp,
-      ArchiveWriter aw, String authority, SortedSet<String> directories) {
-    Logger logger =
-        Logger.getLogger(RelayDescriptorDownloader.class.getName());
-    // TODO use aw
-    try {
-      rdp.initialize();
-    } catch (IOException e) {
-      return;
+      ArchiveWriter aw, List<String> authorities,
+      SortedSet<String> directories) {
+    Logger logger = Logger.getLogger(
+        RelayDescriptorDownloader.class.getName());
+    List<String> remainingAuthorities =
+        new ArrayList<String>(authorities);
+    if (rdp != null) {
+      try {
+        rdp.initialize(); // TODO get rid of this non-sense
+      } catch (IOException e) {
+        return;
+      }
     }
-    try {
-      logger.info("Downloading current consensus from " + authority
-          + "...");
-      URL u = new URL("http://"; + authority
-          + "/tor/status-vote/current/consensus");
-      HttpURLConnection huc = (HttpURLConnection) u.openConnection();
-      huc.setRequestMethod("GET");
-      huc.connect();
-      int response = huc.getResponseCode();
-      if (response == 200) {
-        BufferedInputStream in = new BufferedInputStream(
-            huc.getInputStream());
-        StringBuilder sb = new StringBuilder();
-        int len;
-        byte[] data = new byte[1024];
-        while ((len = in.read(data, 0, 1024)) >= 0) {
-          sb.append(new String(data, 0, len));
-        }
-        in.close();
-        String consensus = sb.toString();
-        rdp.parse(new BufferedReader(new StringReader(consensus)));
+    Set<String> urls = new HashSet<String>();
+    Set<String> downloaded = new HashSet<String>();
+    if (rdp != null) {
+      urls.addAll(rdp.getMissingDescriptorUrls());
+    }
+    do {
+      if (aw != null) {
+        urls.addAll(aw.getMissingDescriptorUrls());
       }
-      logger.info("Finished downloading current consensus from "
-          + authority + ".");
-      logger.info("Downloading extra-info descriptors from " + authority
-          + "...");
-      Stack<String> extraInfos = new Stack<String>();
-      for (String fingerprint : directories) {
-        u = new URL("http://"; + authority + "/tor/extra/fp/"
-            + fingerprint);
-        huc = (HttpURLConnection) u.openConnection();
-        huc.setRequestMethod("GET");
-        huc.connect();
-        response = huc.getResponseCode();
-        if (response == 200) {
-          BufferedInputStream in = new BufferedInputStream(
-              huc.getInputStream());
-          StringBuilder sb = new StringBuilder();
-          int len;
-          byte[] data = new byte[1024];
-          while ((len = in.read(data, 0, 1024)) >= 0) {
-            sb.append(new String(data, 0, len));
+      urls.removeAll(downloaded);
+      SortedSet<String> sortedUrls = new TreeSet<String>(urls);
+      while (!remainingAuthorities.isEmpty() && !sortedUrls.isEmpty()) {
+        String authority = remainingAuthorities.get(0);
+        String url = sortedUrls.first();
+        try {
+          URL u = new URL("http://"; + authority + url);
+          HttpURLConnection huc =
+              (HttpURLConnection) u.openConnection();
+          huc.setRequestMethod("GET");
+          huc.connect();
+          int response = huc.getResponseCode();
+          logger.fine("Downloading http://"; + authority + url + " -> "
+              + response);
+          if (response == 200) {
+            BufferedInputStream in = new BufferedInputStream(
+                huc.getInputStream());
+            StringBuilder sb = new StringBuilder();
+            int len;
+            byte[] data = new byte[1024];
+            while ((len = in.read(data, 0, 1024)) >= 0) {
+              sb.append(new String(data, 0, len));
+            }
+            in.close();
+            String result = sb.toString();
+            if (rdp != null) {
+              BufferedReader br = new BufferedReader(new StringReader(
+                  result));
+              rdp.parse(br);
+              br.close();
+            }
+            if (aw != null) {
+              BufferedReader br = new BufferedReader(new StringReader(
+                  result));
+              try {
+                aw.store(br);
+              } catch (Exception e) {
+                e.printStackTrace();
+                //TODO find better way to handle this
+              }
+              br.close();
+            }
           }
-          in.close();
-          String extraInfo = sb.toString();
-          if (extraInfo.length() > 0) {
-            BufferedReader br = new BufferedReader(
-                new StringReader(extraInfo));
-            rdp.parse(br);
+          sortedUrls.remove(url);
+        } catch (IOException e) {
+          remainingAuthorities.remove(authority);
+          if (!remainingAuthorities.isEmpty()) {
+            logger.log(Level.INFO, "Failed downloading from "
+                + authority + "!", e);
+          } else {
+            logger.log(Level.WARNING, "Failed downloading from "
+                + authority + "! We have no authorities left to download "
+                + "from!", e);
           }
         }
       }
-      logger.info("Finished downloading extra-info descriptors from "
-          + authority + ".");
-    } catch (IOException e) {
-      logger.log(Level.WARNING, "Failed downloading either current "
-          + "consensus or extra-info descriptors from " + authority
-          + "!", e);
-    }
+      downloaded.addAll(urls);
+    } while (!urls.isEmpty());
   }
 }
 
diff --git a/src/RelayDescriptorParser.java b/src/RelayDescriptorParser.java
index e8b29fe..2139f48 100644
--- a/src/RelayDescriptorParser.java
+++ b/src/RelayDescriptorParser.java
@@ -66,6 +66,10 @@ public class RelayDescriptorParser {
   }
   public void parse(BufferedReader br) throws IOException {
     String line = br.readLine();
+    if (line == null) {
+      this.logger.warning("Parsing empty file?");
+      return;
+    }
     if (line.equals("network-status-version 3")) {
       int exit = 0, fast = 0, guard = 0, running = 0, stable = 0;
       String validAfter = null;
@@ -162,15 +166,19 @@ public class RelayDescriptorParser {
     format.setTimeZone(TimeZone.getTimeZone("UTC"));
     long now = System.currentTimeMillis();
     for (String directory : this.directories) {
-      try {
-        long statsEnd = format.parse(this.lastParsedExtraInfos.get(
-            directory)).getTime();
-        if (statsEnd + 36L * 60L * 60L * 1000L < now) {
-          urls.add("/tor/extra/fp/" + directory);
+      if (!this.lastParsedExtraInfos.containsKey(directory)) {
+        urls.add("/tor/extra/fp/" + directory);
+      } else {
+        try {
+          long statsEnd = format.parse(this.lastParsedExtraInfos.get(
+              directory)).getTime();
+          if (statsEnd + 36L * 60L * 60L * 1000L < now) {
+            urls.add("/tor/extra/fp/" + directory);
+          }
+        } catch (ParseException e) {
+          this.logger.log(Level.WARNING, "Failed parsing timestamp in "
+              + this.statsDir + "/relay-descriptor-parse-history!", e);
         }
-      } catch (ParseException e) {
-        this.logger.log(Level.WARNING, "Failed parsing timestamp in "
-            + this.statsDir + "/relay-descriptor-parse-history!", e);
       }
     }
     return urls;
-- 
1.6.5