[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] [ernie/master] Move missing-descriptors logic to descriptor downloader.



Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed, 17 Mar 2010 22:50:51 +0100
Subject: Move missing-descriptors logic to descriptor downloader.
Commit: bdd30a8b44702d5111a45b47bc720902886705f7

---
 config                               |    3 -
 src/ArchiveWriter.java               |  526 ++++------------------------------
 src/CachedRelayDescriptorReader.java |   19 +--
 src/Configuration.java               |   16 -
 src/Main.java                        |   62 +++--
 src/RelayDescriptorDownloader.java   |  482 ++++++++++++++++++++++++++-----
 src/RelayDescriptorParser.java       |  478 +++++++++++++++++--------------
 7 files changed, 767 insertions(+), 819 deletions(-)

diff --git a/config b/config
index 5c202f1..18c5ed6 100644
--- a/config
+++ b/config
@@ -49,9 +49,6 @@
 ## Write directory archives to disk
 #WriteDirectoryArchives 0
 
-## V3 directory authority fingerprints
-#V3DirectoryAuthorities 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,D586D18309DED4CD6D57C18FDB97EFA96D330566,585769C78764D58426B8B52B6651A5A71137189A,27B6B5996C426270A5C95488AA5BCEB6BCC86956,80550987E1D626E3EBA5E5E75A458DE0626D088C,ED03BB616EB2F60BEC80151114BB25CEF515B226,81349FC1F2DBA2C2C11B45CB9706637D480AB913,E2A2AF570166665D738736D0DD58169CC61D8A8B
-
 ## Read cached-* files from a local Tor client
 #ImportCachedRelayDescriptors 1
 
diff --git a/src/ArchiveWriter.java b/src/ArchiveWriter.java
index 63a8477..628025c 100644
--- a/src/ArchiveWriter.java
+++ b/src/ArchiveWriter.java
@@ -1,488 +1,72 @@
 import java.io.*;
-import java.text.*;
-import java.util.*;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
 import java.util.logging.*;
-import org.apache.commons.codec.digest.*;
-import org.apache.commons.codec.binary.*;
 
 public class ArchiveWriter {
-  private SortedSet<String> v3DirectoryAuthorities;
-  private File missingDescriptorsFile;
-  private SortedSet<String> missingDescriptors;
-  private boolean missingDescriptorsFileModified = false;
   private Logger logger;
-  public ArchiveWriter(SortedSet<String> v3DirectoryAuthorities) {
-    this.v3DirectoryAuthorities = v3DirectoryAuthorities;
-    this.missingDescriptorsFile = new File(
-        "stats/archive-writer-parse-history");
+  public ArchiveWriter() {
     this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
-    SimpleDateFormat parseFormat =
-        new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-    parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    this.missingDescriptors = new TreeSet<String>();
-    SimpleDateFormat consensusVoteFormat =
-        new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
-    consensusVoteFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    SimpleDateFormat descriptorFormat =
-        new SimpleDateFormat("yyyy/MM/");
-    descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    if (this.missingDescriptorsFile.exists()) {
-      this.logger.fine("Reading file "
-          + this.missingDescriptorsFile.getAbsolutePath() + "...");
-      try {
-        BufferedReader br = new BufferedReader(new FileReader(
-            this.missingDescriptorsFile));
-        String line = null;
-        long now = System.currentTimeMillis();
-        while ((line = br.readLine()) != null) {
-          // only add to download list if descriptors are still available
-          // on directories
-          long published = parseFormat.parse(line.split(",")[2]).
-              getTime();
-          if (line.startsWith("consensus") &&
-              published + 55L * 60L * 1000L > now) {
-            File consensusFile = new File("directory-archive/consensus/"
-                + consensusVoteFormat.format(new Date(published))
-                + "-consensus");
-            if (!consensusFile.exists()) {
-              this.logger.finer("Initializing missing list with "
-                  + "consensus: valid-after=" + line.split(",")[2]
-                  + ", filename=directory-archive/consensus/"
-                  + consensusVoteFormat.format(new Date(published))
-                  + "-consensus");
-              this.missingDescriptors.add(line);
-            }
-          } else if (line.startsWith("vote") &&
-              published + 55L * 60L * 1000L > now) {
-              // TODO is vote even available for 55 minutes after its
-              // publication?
-            File voteFile = new File("directory-archive/vote/"
-                + consensusVoteFormat.format(new Date(published))
-                + "-vote-" + line.split(",")[1]);
-            File voteFileDir = voteFile.getParentFile();
-            String voteFileName = voteFile.getName();
-            boolean voteFileFound = false;
-            if (voteFileDir.exists()) {
-              for (File f : Arrays.asList(voteFileDir.listFiles())) {
-                if (f.getName().startsWith(voteFileName)) {
-                  voteFileFound = true;
-                  break;
-                }
-              }
-            }
-            if (!voteFileFound) {
-              this.logger.finer("Initializing missing list with vote: "
-                  + "fingerprint=" + line.split(",")[1]
-                  + ", valid-after="
-                  + consensusVoteFormat.format(new Date(published))
-                  + ", filename=directory-archive/vote/"
-                  + consensusVoteFormat.format(new Date(published))
-                  + "-vote-" + line.split(",")[1] + "-*");
-              this.missingDescriptors.add(line);
-            }
-          } else if ((line.startsWith("server") ||
-              line.startsWith("extra")) &&
-              published + 24L * 60L * 60L * 1000L > now) {
-              // TODO are 24 hours okay?
-            boolean isServerDesc = line.startsWith("server");
-            String digest = line.split(",")[1].toLowerCase();
-            File descriptorFile = new File("directory-archive/"
-                + (isServerDesc ? "server-descriptor" : "extra-info")
-                + "/" + descriptorFormat.format(new Date(published))
-                + digest.substring(0, 1) + "/" + digest.substring(1, 2)
-                + "/" + digest);
-            if (!descriptorFile.exists()) {
-              this.logger.finer("Initializing missing list with "
-                  + (isServerDesc ? "server" : "extra-info")
-                  + " descriptor: digest=" + digest
-                  + ", filename=directory-archive/server-descriptor/"
-                  + descriptorFormat.format(new Date(published))
-                  + line.split(",")[1].substring(0, 1) + "/"
-                  + line.split(",")[1].substring(1, 2) + "/"
-                  + line.split(",")[1]);
-              this.missingDescriptors.add(line);
-            }
-          }
-        }
-        br.close();
-        this.logger.fine("Finished reading file "
-            + this.missingDescriptorsFile.getAbsolutePath() + ".");
-      } catch (ParseException e) {
-        this.logger.log(Level.WARNING, "Failed reading file "
-            + this.missingDescriptorsFile.getAbsolutePath()
-            + "! This means that we might forget to dowload descriptors "
-            + "we are missing.", e);
-      } catch (IOException e) {
-        this.logger.log(Level.WARNING, "Failed reading file "
-            + this.missingDescriptorsFile.getAbsolutePath()
-            + "! This means that we might forget to dowload descriptors "
-            + "we are missing.", e);
-      }
-    }
-    // add current consensus and votes to list
-    SimpleDateFormat consensusFormat =
-        new SimpleDateFormat("yyyy-MM-dd HH");
-    consensusFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    String nowConsensusFormat = consensusFormat.format(new Date())
-        + ":00:00";
-    long nowConsensus = (System.currentTimeMillis() / (60L * 60L * 1000L))
-        * (60L * 60L * 1000L);
-    for (String authority : this.v3DirectoryAuthorities) {
-      File voteFile = new File("directory-archive/vote/"
-          + consensusVoteFormat.format(new Date(nowConsensus))
-          + "-vote-" + authority);
-      if (!this.missingDescriptors.contains("vote," + authority + ","
-          + nowConsensusFormat)) {
-        File voteFileDir = voteFile.getParentFile();
-        String voteFileName = voteFile.getName();
-        boolean voteFileFound = false;
-        if (voteFileDir.exists()) {
-          for (File f : Arrays.asList(voteFileDir.listFiles())) {
-            if (f.getName().startsWith(voteFileName)) {
-              voteFileFound = true;
-              break;
-            }
-          }
-        }
-        if (!voteFileFound) {
-          this.logger.finer("Adding vote to missing list: fingerprint="
-              + authority + ", valid-after="
-              + consensusVoteFormat.format(new Date(nowConsensus))
-              + ", filename=directory-archive/vote/"
-              + consensusVoteFormat.format(new Date(nowConsensus))
-              + "-vote-" + authority + "-*");
-          this.missingDescriptors.add("vote," + authority + ","
-              + nowConsensusFormat);
-          this.missingDescriptorsFileModified = true;
-        }
-      }
-    }
-    File consensusFile = new File("directory-archive/consensus/"
-        + consensusVoteFormat.format(new Date(nowConsensus))
-        + "-consensus");
-    if (!this.missingDescriptors.contains("consensus,NA,"
-        + nowConsensusFormat) && !consensusFile.exists()) {
-      this.logger.finer("Adding consensus to missing list: valid-after="
-          + nowConsensusFormat
-          + ", filename=directory-archive/consensus/"
-          + consensusVoteFormat.format(new Date(nowConsensus))
-          + "-consensus");
-      this.missingDescriptors.add("consensus,NA,"
-          + nowConsensusFormat);
-      this.missingDescriptorsFileModified = true;
-    }
   }
-  public void store(byte[] data) throws IOException, ParseException {
-    BufferedReader br = new BufferedReader(new StringReader(new String(
-        data, "US-ASCII")));
-    String line = br.readLine();
-    if (line == null) {
-      this.logger.warning("Someone gave us an empty file for storing!");
-      return;
-    }
-    StringBuilder sb = new StringBuilder();
-    sb.append(line + "\n");
-    SimpleDateFormat parseFormat =
-        new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-    parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    if (line.equals("network-status-version 3")) {
-      // TODO when parsing the current consensus, check the fresh-until
-      // time to see when we switch from hourly to half-hourly
-      // consensuses; in that case, add next half-hourly consensus to
-      // missing list and warn!
-      boolean isConsensus = true;
-      String validAfterTime = null;
-      long validAfter = -1L;
-      long now = System.currentTimeMillis();
-      String fingerprint = null;
-      SimpleDateFormat descriptorFormat =
-          new SimpleDateFormat("yyyy/MM/");
-      descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-      while ((line = br.readLine()) != null) {
-        sb.append(line + "\n");
-        if (line.equals("vote-status vote")) {
-          isConsensus = false;
-        } else if (line.startsWith("valid-after ")) {
-          validAfterTime = line.substring("valid-after ".length());
-          validAfter = parseFormat.parse(validAfterTime).getTime();
-        } else if (line.startsWith("dir-source ") &&
-            !this.v3DirectoryAuthorities.contains(
-            line.split(" ")[2]) && validAfter + 55L * 60L * 1000L > now) {
-          this.logger.warning("Unknown v3 directory authority fingerprint "
-              + "in consensus line '" + line + "'. You should update your "
-              + "V3DirectoryAuthorities config option!");
-          fingerprint = line.split(" ")[2];
-          long nowConsensus = (now / (60L * 60L * 1000L))
-              * (60L * 60L * 1000L);
-          SimpleDateFormat consensusVoteFormat =
-              new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
-          consensusVoteFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-          File voteFile = new File("directory-archive/vote/"
-                + consensusVoteFormat.format(new Date(nowConsensus))
-                + "-vote-" + fingerprint);
-          if (!this.missingDescriptors.contains("vote," + fingerprint
-              + "," + parseFormat.format(new Date(nowConsensus)))) {
-            File voteFileDir = voteFile.getParentFile();
-            String voteFileName = voteFile.getName();
-            boolean voteFileFound = false;
-            if (voteFileDir.exists()) {
-              for (File f : Arrays.asList(voteFileDir.listFiles())) {
-                if (f.getName().startsWith(voteFileName)) {
-                  voteFileFound = true;
-                  break;
-                }
-              }
-            }
-            if (!voteFileFound) {
-              this.logger.finer("Adding vote to missing list: "
-                  + "fingerprint=" + fingerprint + ", valid-after="
-                  + parseFormat.format(new Date(nowConsensus))
-                  + ", filename=directory-archive/vote/"
-                  + consensusVoteFormat.format(new Date(nowConsensus))
-                  + "-vote-" + fingerprint + "-*");
-              this.missingDescriptors.add("vote," + fingerprint + ","
-                  + parseFormat.format(new Date(nowConsensus)));
-              this.missingDescriptorsFileModified = true;
-            }
-          }
-        } else if (line.startsWith("fingerprint ")) {
-          fingerprint = line.split(" ")[1];
-        } else if (line.startsWith("r ")) {
-          String publishedTime = line.split(" ")[4] + " "
-              + line.split(" ")[5];
-          long published = parseFormat.parse(publishedTime).getTime();
-          String serverDesc = Hex.encodeHexString(Base64.decodeBase64(
-              line.split(" ")[3] + "=")).toLowerCase();
-          // TODO are 24 hours okay?
-          File descriptorFile = new File(
-              "directory-archive/server-descriptor/"
-              + descriptorFormat.format(new Date(published))
-              + serverDesc.substring(0, 1) + "/"
-              + serverDesc.substring(1, 2) + "/" + serverDesc);
-          if (published + 24L * 60L * 60L * 1000L > now &&
-              !this.missingDescriptors.contains("server," + serverDesc
-                + "," + publishedTime) && !descriptorFile.exists()) {
-            this.logger.finer("Adding server descriptor to missing list: "
-                + "digest=" + serverDesc
-                + ", filename=directory-archive/server-descriptor/"
-                + descriptorFormat.format(new Date(published))
-                + serverDesc.substring(0, 1) + "/"
-                + serverDesc.substring(1, 2) + "/" + serverDesc);
-            this.missingDescriptors.add("server," + serverDesc + ","
-                + publishedTime);
-            this.missingDescriptorsFileModified = true;
-          }
-        }
-      }
-      SimpleDateFormat printFormat =
-          new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
-      printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-      if (isConsensus) {
-        File consensusFile = new File("directory-archive/consensus/"
-            + printFormat.format(new Date(validAfter)) + "-consensus");
-        if (!consensusFile.exists()) {
-          this.logger.finer("Storing consensus: valid-after="
-              + validAfterTime + ", filename=directory-archive/consensus/"
-              + printFormat.format(new Date(validAfter)) + "-consensus");
-          consensusFile.getParentFile().mkdirs();
-          BufferedOutputStream bos = new BufferedOutputStream(
-              new FileOutputStream(consensusFile));
-          bos.write(data, 0, data.length);
-          bos.close();
-          this.logger.finer("Removing consensus from missing list: "
-              + "valid-after=" + validAfterTime
-              + ", filename=directory-archive/consensus/"
-              + printFormat.format(new Date(validAfter)) + "-consensus");
-          this.missingDescriptors.remove("consensus,NA,"
-              + validAfterTime);
-          this.missingDescriptorsFileModified = true;
-        } else {
-          this.logger.finer("Not storing consensus, because we already "
-              + "have it: valid-after=" + validAfterTime
-              + ", filename=directory-archive/consensus/"
-              + printFormat.format(new Date(validAfter)) + "-consensus");
-        }
-      } else {
-        String ascii = new String(data, "US-ASCII");
-        String startToken = "network-status-version ";
-        String sigToken = "directory-signature ";
-        int start = ascii.indexOf(startToken);
-        int sig = ascii.indexOf(sigToken);
-        if (start < 0 || sig < 0 || sig < start) {
-          this.logger.warning("Cannot determine vote digest! Skipping.");
-          return;
-        }
-        sig += sigToken.length();
-        byte[] forDigest = new byte[sig - start];
-        System.arraycopy(data, start, forDigest, 0, sig - start);
-        String digest = DigestUtils.shaHex(forDigest).toUpperCase();
-        File voteFile = new File("directory-archive/vote/"
-            + printFormat.format(new Date(validAfter)) + "-vote-"
-            + fingerprint + "-" + digest);
-        if (!voteFile.exists()) {
-          this.logger.finer("Storing vote: fingerprint=" + fingerprint
-              + ", valid-after="
-              + printFormat.format(new Date(validAfter))
-              + ", filename=directory-archive/vote/"
-              + printFormat.format(new Date(validAfter)) + "-vote-"
-              + fingerprint + "-" + digest);
-          voteFile.getParentFile().mkdirs();
-          BufferedOutputStream bos = new BufferedOutputStream(
-              new FileOutputStream(voteFile));
-          bos.write(data, 0, data.length);
-          bos.close();
-          this.logger.finer("Removing vote from missing list: "
-              + "fingerprint=" + fingerprint + ", valid-after="
-              + printFormat.format(new Date(validAfter))
-              + ", filename=directory-archive/vote/"
-              + printFormat.format(new Date(validAfter)) + "-vote-"
-              + fingerprint + "-" + digest);
-          this.missingDescriptors.remove("vote," + fingerprint + ","
-              + validAfterTime);
-          this.missingDescriptorsFileModified = true;
-        } else {
-          this.logger.finer("Not storing vote, because we already have "
-              + "it: fingerprint=" + fingerprint + ", valid-after="
-              + printFormat.format(new Date(validAfter))
-              + ", filename=directory-archive/vote/"
-              + printFormat.format(new Date(validAfter)) + "-vote-"
-              + fingerprint + "-" + digest);
-        }
-      }
-    } else if (line.startsWith("router ") ||
-        line.startsWith("extra-info ")) {
-      boolean isServerDescriptor = line.startsWith("router ");
-      String publishedTime = null;
-      long published = -1L;
-      while ((line = br.readLine()) != null) {
-        if (line.startsWith("published ")) {
-          publishedTime = line.substring("published ".length());
-          published = parseFormat.parse(publishedTime).getTime();
-        } else if (line.startsWith("opt extra-info-digest ") ||
-            line.startsWith("extra-info-digest ")) {
-          String extraInfoDigest = line.startsWith("opt ") ?
-              line.split(" ")[2].toLowerCase() :
-              line.split(" ")[1].toLowerCase();
-          SimpleDateFormat descriptorFormat =
-              new SimpleDateFormat("yyyy/MM/");
-          descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-          File descriptorFile = new File("directory-archive/extra-info/"
-              + descriptorFormat.format(new Date(published))
-              + extraInfoDigest.substring(0, 1) + "/"
-              + extraInfoDigest.substring(1, 2) + "/"
-              + extraInfoDigest);
-          if (!this.missingDescriptors.contains("extra,"
-              + extraInfoDigest + "," + publishedTime) &&
-              !descriptorFile.exists()) {
-            this.logger.finer("Adding extra-info descriptor to missing "
-                + "list: digest=" + extraInfoDigest
-                + ", filename=directory-archive/extra-info/"
-                + descriptorFormat.format(new Date(published))
-                + extraInfoDigest.substring(0, 1) + "/"
-                + extraInfoDigest.substring(1, 2) + "/"
-                + extraInfoDigest);
-            this.missingDescriptors.add("extra," + extraInfoDigest + ","
-                + publishedTime);
-            this.missingDescriptorsFileModified = true;
-          }
-        }
-      }
-      String ascii = new String(data, "US-ASCII");
-      String startToken = isServerDescriptor ?
-          "router " : "extra-info ";
-      String sigToken = "\nrouter-signature\n";
-      int start = ascii.indexOf(startToken);
-      int sig = ascii.indexOf(sigToken) + sigToken.length();
-      if (start < 0 || sig < 0 || sig < start) {
-        this.logger.warning("Cannot determine descriptor digest! "
-            + "Skipping.");
-        return;
-      }
-      byte[] forDigest = new byte[sig - start];
-      System.arraycopy(data, start, forDigest, 0, sig - start);
-      String digest = DigestUtils.shaHex(forDigest);
-      SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
-      printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-      File descriptorFile = new File("directory-archive/"
-          + (isServerDescriptor ? "server-descriptor" : "extra-info") + "/"
-          + printFormat.format(new Date(published))
-          + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
-          + digest);
-      if (!descriptorFile.exists()) {
-        this.logger.finer("Storing " + (isServerDescriptor ?
-            "server descriptor" : "extra-info descriptor")
-            + ": digest=" + digest + ", filename=directory-archive/"
-            + (isServerDescriptor ? "server-descriptor" : "extra-info")
-            + "/" + printFormat.format(new Date(published))
-            + digest.substring(0, 1) + "/" + digest.substring(1, 2)
-            + "/" + digest);
-        descriptorFile.getParentFile().mkdirs();
+
+  private void store(byte[] data, String filename) {
+    try {
+      File file = new File(filename);
+      if (!file.exists()) {
+        this.logger.finer("Storing " + filename);
+        file.getParentFile().mkdirs();
         BufferedOutputStream bos = new BufferedOutputStream(
-            new FileOutputStream(descriptorFile));
+            new FileOutputStream(file));
         bos.write(data, 0, data.length);
         bos.close();
-        this.logger.finer("Removing " + (isServerDescriptor ?
-            "server descriptor" : "extra-info descriptor")
-            + " from missing list: digest=" + digest
-            + ", filename=directory-archive/"
-            + (isServerDescriptor ? "server-descriptor" : "extra-info")
-            + "/" + printFormat.format(new Date(published))
-            + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
-            + digest);
-        if (isServerDescriptor) {
-          this.missingDescriptors.remove("server," + digest + ","
-              + publishedTime);
-        } else {
-          this.missingDescriptors.remove("extra," + digest + ","
-              + publishedTime);
-        }
-        this.missingDescriptorsFileModified = true;
-      } else {
-        this.logger.finer("Not storing " + (isServerDescriptor ?
-            "server descriptor" : "extra-info descriptor")
-            + ", because we already have it: digest=" + digest
-            + ", filename=directory-archive/"
-            + (isServerDescriptor ? "server-descriptor" : "extra-info")
-            + "/" + printFormat.format(new Date(published))
-            + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
-            + digest);
       }
+    } catch (IOException e) {
+      // TODO handle
     }
   }
-  public Set<String> getMissingDescriptorUrls() {
-    Set<String> urls = new HashSet<String>();
-    for (String line : this.missingDescriptors) {
-      if (line.startsWith("consensus,")) {
-        urls.add("/tor/status-vote/current/consensus");
-      } else if (line.startsWith("vote,")) {
-        urls.add("/tor/status-vote/current/" + line.split(",")[1]);
-      } else if (line.startsWith("server,")) {
-        urls.add("/tor/server/d/" + line.split(",")[1]);
-      } else if (line.startsWith("extra,")) {
-        urls.add("/tor/extra/d/" + line.split(",")[1]);
-      }
-    }
-    return urls;
+
+  public void storeConsensus(byte[] data, long validAfter) {
+    SimpleDateFormat printFormat = new SimpleDateFormat(
+        "yyyy/MM/yyyy-MM-dd-HH-mm-ss");
+    printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    String filename = "directory-archive/consensus/"
+        + printFormat.format(new Date(validAfter)) + "-consensus";
+    this.store(data, filename);
   }
-  public void writeFile() {
-    if (this.missingDescriptorsFileModified) {
-      try {
-        this.logger.fine("Writing file "
-            + this.missingDescriptorsFile.getAbsolutePath() + "...");
-        this.missingDescriptorsFile.getParentFile().mkdirs();
-        BufferedWriter bw = new BufferedWriter(new FileWriter(
-            this.missingDescriptorsFile));
-        for (String line : this.missingDescriptors) {
-          bw.write(line + "\n");
-        }
-        bw.close();
-        this.logger.fine("Finished writing file "
-            + this.missingDescriptorsFile.getAbsolutePath() + ".");
-      } catch (IOException e) {
-        this.logger.log(Level.WARNING, "Failed writing "
-            + this.missingDescriptorsFile.getAbsolutePath() + "!", e);
-      }
-    }
+
+  public void storeVote(byte[] data, long validAfter,
+      String fingerprint, String digest) {
+    SimpleDateFormat printFormat = new SimpleDateFormat(
+        "yyyy/MM/yyyy-MM-dd-HH-mm-ss");
+    printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    String filename = "directory-archive/vote/"
+        + printFormat.format(new Date(validAfter)) + "-vote-"
+        + fingerprint + "-" + digest;
+    this.store(data, filename);
+  }
+
+  public void storeServerDescriptor(byte[] data, String digest,
+      long published) {
+    SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
+    printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    String filename = "directory-archive/server-descriptor/"
+        + printFormat.format(new Date(published))
+        + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
+        + digest;
+    this.store(data, filename);
+  }
+
+  public void storeExtraInfoDescriptor(byte[] data,
+      String extraInfoDigest, long published) {
+    SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
+    descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    String filename = "directory-archive/extra-info/"
+        + descriptorFormat.format(new Date(published))
+        + extraInfoDigest.substring(0, 1) + "/"
+        + extraInfoDigest.substring(1, 2) + "/"
+        + extraInfoDigest;
+    this.store(data, filename);
   }
 }
 
diff --git a/src/CachedRelayDescriptorReader.java b/src/CachedRelayDescriptorReader.java
index 28a979c..89fa3fe 100644
--- a/src/CachedRelayDescriptorReader.java
+++ b/src/CachedRelayDescriptorReader.java
@@ -1,15 +1,12 @@
 import java.io.*;
-import java.text.*;
 import java.util.logging.*;
-//import org.apache.commons.codec.digest.*; TODO currently unused
 
 /**
  * Parses all descriptors in local directory cacheddesc/ and sorts them
  * into directory structure in directory-archive/.
  */
 public class CachedRelayDescriptorReader {
-  public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
-      ArchiveWriter aw) {
+  public CachedRelayDescriptorReader(RelayDescriptorParser rdp) {
     // TODO check if files are stale; print out warning that Tor process
     // might have died
     Logger logger = Logger.getLogger(
@@ -32,9 +29,6 @@ public class CachedRelayDescriptorReader {
           bis.close();
           byte[] allData = baos.toByteArray();
           if (f.getName().equals("cached-consensus")) {
-            if (aw != null) {
-              aw.store(allData);
-            }
             if (rdp != null) {
               rdp.parse(allData);
             }
@@ -62,16 +56,8 @@ public class CachedRelayDescriptorReader {
                 break;
               }
               end += endToken.length();
-              /* String desc = ascii.substring(start, end);
-              byte[] forDigest = new byte[sig - start];
-              System.arraycopy(allData, start, forDigest, 0, sig - start);
-              String digest = DigestUtils.shaHex(forDigest);
-              TODO this stuff is unused? */
               byte[] descBytes = new byte[end - start];
               System.arraycopy(allData, start, descBytes, 0, end - start);
-              if (aw != null) {
-                aw.store(descBytes);
-              }
               if (rdp != null) {
                 rdp.parse(descBytes);
               }
@@ -81,9 +67,6 @@ public class CachedRelayDescriptorReader {
         } catch (IOException e) {
           logger.log(Level.WARNING, "Failed reading cacheddesc/ "
               + "directory.", e);
-        } catch (ParseException e) {
-          logger.log(Level.WARNING, "Failed reading cacheddesc/ "
-              + "directory.", e);
         }
       }
     }
diff --git a/src/Configuration.java b/src/Configuration.java
index 4ba3044..cf8c7d1 100644
--- a/src/Configuration.java
+++ b/src/Configuration.java
@@ -23,16 +23,6 @@ public class Configuration {
   private List<String> relayPlatforms = new ArrayList<String>(Arrays.asList(
       "Linux,Windows,Darwin,FreeBSD".split(",")));
   private boolean writeDirectoryArchives = false;
-  private SortedSet<String> v3DirectoryAuthorities = new TreeSet<String>(
-      Arrays.asList(("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,"
-      + "E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,"
-      + "D586D18309DED4CD6D57C18FDB97EFA96D330566,"
-      + "585769C78764D58426B8B52B6651A5A71137189A,"
-      + "27B6B5996C426270A5C95488AA5BCEB6BCC86956,"
-      + "80550987E1D626E3EBA5E5E75A458DE0626D088C,"
-      + "ED03BB616EB2F60BEC80151114BB25CEF515B226,"
-      + "81349FC1F2DBA2C2C11B45CB9706637D480AB913,"
-      + "E2A2AF570166665D738736D0DD58169CC61D8A8B").split(",")));
   private boolean importCachedRelayDescriptors = true;
   private boolean importDirectoryArchives = true;
   private boolean importSanitizedBridges = true;
@@ -91,9 +81,6 @@ public class Configuration {
         } else if (line.startsWith("WriteDirectoryArchives")) {
           this.writeDirectoryArchives = Integer.parseInt(
               line.split(" ")[1]) != 0;
-        } else if (line.startsWith("V3DirectoryAuthorities")) {
-          this.v3DirectoryAuthorities = new TreeSet<String>(
-              Arrays.asList(line.split(" ")[1].split(",")));
         } else if (line.startsWith("ImportCachedRelayDescriptors")) {
           this.importCachedRelayDescriptors = Integer.parseInt(
               line.split(" ")[1]) != 0;
@@ -191,9 +178,6 @@ public class Configuration {
   public boolean getWriteDirectoryArchives() {
     return this.writeDirectoryArchives;
   }
-  public SortedSet<String> getV3DirectoryAuthorities() {
-    return this.v3DirectoryAuthorities;
-  }
   public boolean getImportCachedRelayDescriptors() {
     return this.importCachedRelayDescriptors;
   }
diff --git a/src/Main.java b/src/Main.java
index 2d96396..fa40fa0 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -38,40 +38,50 @@ public class Main {
         new ServerDescriptorStatsFileHandler(config.getRelayVersions(),
         config.getRelayPlatforms()) : null;
 
-    // Prepare relay descriptor parser (only if we are writing the
-    // stats)
+    // Prepare writing relay descriptor archive to disk
+    ArchiveWriter aw = config.getWriteDirectoryArchives() ?
+        new ArchiveWriter() : null;
+
+    // Prepare relay descriptor parser (only if we are writing stats or
+    // directory archives to disk)
     RelayDescriptorParser rdp = config.getWriteConsensusStats() ||
         config.getWriteBridgeStats() || config.getWriteDirreqStats() ||
-        config.getWriteServerDescriptorStats() ?
-        new RelayDescriptorParser(csfh, bsfh, dsfh, sdsfh, countries,
+        config.getWriteServerDescriptorStats() ||
+        config.getWriteDirectoryArchives() ?
+        new RelayDescriptorParser(csfh, bsfh, dsfh, sdsfh, aw, countries,
         directories) : null;
 
-    // Prepare writing relay descriptor archive to disk
-    ArchiveWriter aw = config.getWriteDirectoryArchives() ?
-        new ArchiveWriter(config.getV3DirectoryAuthorities()) : null;
-
     // Import/download relay descriptors from the various sources
-    if (config.getImportCachedRelayDescriptors()) {
-      new CachedRelayDescriptorReader(rdp, aw);
-    }
-    if (config.getImportDirectoryArchives()) {
-      new ArchiveReader(rdp, "archives");
-    }
-    if (config.getDownloadRelayDescriptors()) {
-      new RelayDescriptorDownloader(rdp, aw,
-          config.getDownloadFromDirectoryAuthorities(),
-          directories);
+    if (rdp != null) {
+      RelayDescriptorDownloader rdd = null;
+      if (config.getDownloadRelayDescriptors()) {
+        List<String> dirSources =
+            config.getDownloadFromDirectoryAuthorities();
+        boolean downloadCurrentConsensus = aw != null || csfh != null ||
+            bsfh != null || sdsfh != null;
+        boolean downloadCurrentVotes = aw != null;
+        boolean downloadAllServerDescriptors = aw != null || sdsfh != null;
+        boolean downloadAllExtraInfos = aw != null;
+        Set<String> downloadDescriptorsForRelays = directories;
+        rdd = new RelayDescriptorDownloader(rdp, dirSources,
+            downloadCurrentConsensus, downloadCurrentVotes,
+            downloadAllServerDescriptors, downloadAllExtraInfos,
+            downloadDescriptorsForRelays);
+        rdp.setRelayDescriptorDownloader(rdd);
+      }
+      if (config.getImportCachedRelayDescriptors()) {
+        new CachedRelayDescriptorReader(rdp);
+      }
+      if (config.getImportDirectoryArchives()) {
+        new ArchiveReader(rdp, "archives");
+      }
+      if (rdd != null) {
+        rdd.downloadMissingDescriptors();
+        rdd.writeFile();
+      }
     }
 
     // Write output to disk that only depends on relay descriptors
-    if (aw != null) {
-      aw.writeFile();
-      aw = null;
-    }
-    if (rdp != null) {
-      rdp.writeFile();
-      rdp = null;
-    }
     if (dsfh != null) {
       dsfh.writeFile();
       dsfh = null;
diff --git a/src/RelayDescriptorDownloader.java b/src/RelayDescriptorDownloader.java
index a3bc7d3..d7654c5 100644
--- a/src/RelayDescriptorDownloader.java
+++ b/src/RelayDescriptorDownloader.java
@@ -1,38 +1,397 @@
 import java.io.*;
 import java.net.*;
+import java.text.*;
 import java.util.*;
 import java.util.logging.*;
-import org.apache.commons.codec.digest.*;
 
 /**
- * Download the current consensus and relevant extra-info descriptors and
- * hand them to the relay descriptor parser.
+ * Downloads missing relay descriptors from the directories via HTTP.
+ * Keeps a list of missing descriptors that gets updated by parse results
+ * from <code>RelayDescriptorParser</code>. Only descriptors on that
+ * missing list that we think might be available on the directories are
+ * downloaded.
  */
 public class RelayDescriptorDownloader {
+
+  /**
+   * Text file containing the descriptors that we are missing and that we
+   * want to download in <code>downloadMissingDescriptors</code>.
+   * Lines are formatted as:
+   * - "consensus,<validafter>,<parsed>",
+   * - "vote,<validafter>,<fingerprint>,<parsed>",
+   * - "server,<published>,<relayid>,<descid>,<parsed>", or
+   * - "extra,<published>,<relayid>,<descid><parsed>".
+   */
+  private File missingDescriptorsFile;
+
+  /**
+   * Relay descriptors that we are missing and that we want to download
+   * either in this execution or write to disk and try next time. Map keys
+   * contain comma-separated values as in the missing descriptors files
+   * without the parsed column. Map values contain the parsed column.
+   */
+  private SortedMap<String, String> missingDescriptors;
+
+  /**
+   * <code>RelayDescriptorParser</code> that we will hand over the
+   * downloaded descriptors for parsing.
+   */
+  private RelayDescriptorParser rdp;
+
+  /**
+   * Directories that we will try to download missing descriptors from.
+   */
+  private List<String> dirSources;
+
+  /**
+   * Should we try to download the current consensus if we don't have it?
+   */
+  private boolean downloadCurrentConsensus;
+
+  /**
+   * Should we try to download current votes if we don't have them?
+   */
+  private boolean downloadCurrentVotes;
+
+  /**
+   * Should we try to download all missing server descriptors that have
+   * been published within the past 24 hours?
+   */
+  private boolean downloadAllServerDescriptors;
+
+  /**
+   * Should we try to download all missing extra-info descriptors that
+   * have been published within the past 24 hours?
+   */
+  private boolean downloadAllExtraInfos;
+
+  /**
+   * Should we try to download missing server and extra-info descriptors
+   * of certain relays that have been published within the past 24 hours?
+   */
+  private Set<String> downloadDescriptorsForRelays;
+
+  /**
+   * valid-after time that we expect the current consensus and votes to
+   * have, formatted "yyyy-MM-dd HH:mm:ss". We only expect to find
+   * consensuses and votes with this valid-after time on the directories.
+   * This time is initialized as the beginning of the current hour.
+   */
+  private String currentValidAfter;
+
+  /**
+   * Cut-off time for missing server and extra-info descriptors, formatted
+   * "yyyy-MM-dd HH:mm:ss". This time is initialized as the current system
+   * time minus 24 hours.
+   */
+  private String descriptorCutOff;
+
+  /**
+   * Current timestamp that is written to the missing list for descriptors
+   * that we parsed in this execution. This timestamp is most useful for
+   * debugging purposes when looking at the missing list. For execution it
+   * only matters whether the parsed time is "NA" or has some other value.
+   */
+  private String parsedTimestampString;
+
+  /**
+   * Logger for this class.
+   */
+  private Logger logger;
+
+  /**
+   * Initializes this class, including reading in missing descriptors from
+   * <code>stats/missing-relay-descriptors</code>.
+   */
   public RelayDescriptorDownloader(RelayDescriptorParser rdp,
-      ArchiveWriter aw, List<String> authorities,
-      SortedSet<String> directories) {
-    Logger logger = Logger.getLogger(
-        RelayDescriptorDownloader.class.getName());
-    List<String> remainingAuthorities =
-        new ArrayList<String>(authorities);
-    Set<String> urls = new HashSet<String>();
-    Set<String> downloaded = new HashSet<String>();
-    if (rdp != null) {
-      urls.addAll(rdp.getMissingDescriptorUrls());
+      List<String> dirSources, boolean downloadCurrentConsensus,
+      boolean downloadCurrentVotes, boolean downloadAllServerDescriptors,
+      boolean downloadAllExtraInfos,
+      Set<String> downloadDescriptorsForRelays) {
+
+    /* Memorize argument values. */
+    this.rdp = rdp;
+    this.dirSources = dirSources;
+    this.downloadCurrentConsensus = downloadCurrentConsensus;
+    this.downloadCurrentVotes = downloadCurrentVotes;
+    this.downloadAllServerDescriptors = downloadAllServerDescriptors;
+    this.downloadAllExtraInfos = downloadAllExtraInfos;
+    this.downloadDescriptorsForRelays = downloadDescriptorsForRelays;
+
+    /* Initialize logger. */
+    this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
+
+    /* Prepare cut-off times and timestamp for missing descriptors
+     * list. */
+    SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    format.setTimeZone(TimeZone.getTimeZone("UTC"));
+    long now = System.currentTimeMillis();
+    this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) *
+        (60L * 60L * 1000L));
+    this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L);
+    this.parsedTimestampString = format.format(now);
+
+    /* Initialize missing list and put current consensus on it if we want
+     * it. */
+    this.missingDescriptors = new TreeMap<String, String>();
+    if (this.downloadCurrentConsensus) {
+      this.missingDescriptors.put("consensus," + this.currentValidAfter,
+          "NA");
+    }
+
+    /* Read list of missing descriptors from disk and memorize those that
+     * we are interested in and that are likely to be found on the
+     * directory servers. */
+    this.missingDescriptorsFile = new File(
+        "stats/missing-relay-descriptors");
+    if (this.missingDescriptorsFile.exists()) {
+      try {
+        this.logger.fine("Reading file "
+            + this.missingDescriptorsFile.getAbsolutePath() + "...");
+        BufferedReader br = new BufferedReader(new FileReader(
+            this.missingDescriptorsFile));
+        String line = null;
+        while ((line = br.readLine()) != null) {
+          if (line.split(",").length > 2) {
+            String published = line.split(",")[1];
+            if (((line.startsWith("consensus,") ||
+                line.startsWith("vote,")) &&
+                this.currentValidAfter.equals(published)) ||
+                ((line.startsWith("server,") ||
+                line.startsWith("extra,")) &&
+                this.descriptorCutOff.compareTo(published) <= 0)) {
+              int separateAt = line.lastIndexOf(",");
+              this.missingDescriptors.put(line.substring(0,
+                  separateAt), line.substring(separateAt + 1));
+            }
+          } else {
+            this.logger.fine("Invalid line '" + line + "' in "
+                + this.missingDescriptorsFile.getAbsolutePath()
+                + ". Ignoring.");
+          }
+        }
+        br.close();
+        this.logger.fine("Finished reading file "
+            + this.missingDescriptorsFile.getAbsolutePath() + ".");
+      } catch (IOException e) {
+        this.logger.log(Level.WARNING, "Failed to read file "
+            + this.missingDescriptorsFile.getAbsolutePath()
+            + "! This means that we might forget to dowload relay "
+            + "descriptors we are missing.", e);
+      }
+    }
+  }
+
+  /**
+   * We have parsed a consensus. Take this consensus off the missing list
+   * and add the votes created by the given <code>dirSources</code> and
+   * the <code>serverDescriptors</code> in the format
+   * "<published>,<relayid>,<descid>" to that list.
+   */
+  public void haveParsedConsensus(String validAfter,
+      Set<String> dirSources, Set<String> serverDescriptors) {
+
+    /* Mark consensus as parsed. */
+    if (this.currentValidAfter.equals(validAfter)) {
+      String consensusKey = "consensus," + validAfter;
+      this.missingDescriptors.put(consensusKey,
+          this.parsedTimestampString);
+
+      /* Add votes to missing list. */
+      if (this.downloadCurrentVotes) {
+        for (String dirSource : dirSources) {
+          String voteKey = "vote," + validAfter + "," + dirSource;
+          if (!this.missingDescriptors.containsKey(voteKey)) {
+            this.missingDescriptors.put(voteKey, "NA");
+          }
+        }
+      }
+    }
+
+    /* Add server descriptors to missing list. */
+    if (this.downloadAllServerDescriptors ||
+        this.downloadDescriptorsForRelays != null) {
+      for (String serverDescriptor : serverDescriptors) {
+        String published = serverDescriptor.split(",")[0];
+        if (this.descriptorCutOff.compareTo(published) <= 0) {
+          if (this.downloadAllServerDescriptors ||
+              (this.downloadDescriptorsForRelays != null &&
+              this.downloadDescriptorsForRelays.contains(
+              serverDescriptor.split(",")[1].toUpperCase()))) {
+            String serverDescriptorKey = "server," + serverDescriptor;
+            if (!this.missingDescriptors.containsKey(
+                serverDescriptorKey)) {
+              this.missingDescriptors.put(serverDescriptorKey, "NA");
+            }
+          }
+        }
+      }
     }
+  }
+
+  /**
+   * We have parsed a vote. Take this vote off the missing list.
+   */
+  public void haveParsedVote(String validAfter, String fingerprint,
+      Set<String> serverDescriptors) {
+
+    /* Mark consensus as parsed. */
+    if (this.currentValidAfter.equals(validAfter)) {
+      String voteKey = "vote," + validAfter + "," + fingerprint;
+      this.missingDescriptors.put(voteKey, this.parsedTimestampString);
+    }
+
+    /* Add server descriptors to missing list. */
+    if (this.downloadAllServerDescriptors ||
+        this.downloadDescriptorsForRelays != null) {
+      for (String serverDescriptor : serverDescriptors) {
+        String published = serverDescriptor.split(",")[0];
+        if (this.descriptorCutOff.compareTo(published) < 0) {
+          if (this.downloadDescriptorsForRelays == null ||
+              this.downloadDescriptorsForRelays.contains(
+              serverDescriptor.split(",")[1].toUpperCase())) {
+            String serverDescriptorKey = "server," + serverDescriptor;
+            if (!this.missingDescriptors.containsKey(
+                serverDescriptorKey)) {
+              this.missingDescriptors.put(serverDescriptorKey, "NA");
+            }
+          }
+        }
+      }
+    }
+  }
+
+  /**
+   * We have parsed a server descriptor. Take this server descriptor off
+   * the missing list and put the extra-info descriptor digest on that
+   * list.
+   */
+  public void haveParsedServerDescriptor(String published,
+      String relayIdentity, String serverDescriptorDigest,
+      String extraInfoDigest) {
+
+    /* Mark server descriptor as parsed. */
+    if (this.descriptorCutOff.compareTo(published) <= 0) {
+      String serverDescriptorKey = "server," + published + ","
+          + relayIdentity + "," + serverDescriptorDigest;
+      this.missingDescriptors.put(serverDescriptorKey,
+          this.parsedTimestampString);
+
+      /* Add extra-info descriptor to missing list. */
+      if (extraInfoDigest != null && (this.downloadAllExtraInfos ||
+          (this.downloadDescriptorsForRelays != null &&
+          this.downloadDescriptorsForRelays.contains(relayIdentity.
+          toUpperCase())))) {
+        String extraInfoKey = "extra," + published + ","
+            + relayIdentity + "," + extraInfoDigest;
+        if (!this.missingDescriptors.containsKey(extraInfoKey)) {
+          this.missingDescriptors.put(extraInfoKey, "NA");
+        }
+      }
+    }
+  }
+
+  /**
+   * We have parsed an extra-info descriptor. Take it off the missing
+   * list.
+   */
+  public void haveParsedExtraInfoDescriptor(String published,
+      String relayIdentity, String extraInfoDigest) {
+    if (this.descriptorCutOff.compareTo(published) <= 0) {
+      String extraInfoKey = "extra," + published + ","
+          + relayIdentity + "," + extraInfoDigest;
+      this.missingDescriptors.put(extraInfoKey,
+          this.parsedTimestampString);
+    }
+  }
+
+  /**
+   * Downloads missing descriptors that we think might still be available
+   * on the directories.
+   */
+  public void downloadMissingDescriptors() {
+
+    /* Update cut-off times to reflect that execution so far might have
+     * taken a few minutes and that some descriptors aren't available on
+     * the directories anymore. */
+    SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    format.setTimeZone(TimeZone.getTimeZone("UTC"));
+    long now = System.currentTimeMillis();
+    this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) *
+        (60L * 60L * 1000L));
+    this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L);
+
+    /* Remember which directories remain as source for downloading
+     * descriptors. */
+    List<String> remainingDirSources =
+        new ArrayList<String>(this.dirSources);
+
+    /* URLs of descriptors we want to download. */
+    SortedSet<String> urls = new TreeSet<String>();
+
+    /* URLs of descriptors we have downloaded or at least tried to
+     * download. */
+    SortedSet<String> downloaded = new TreeSet<String>();
+
+    /* We might need more than one iteration for downloading descriptors,
+     * because we might learn about descriptors while parsing those that
+     * we got. In every iteration, compile a new list of URLs, remove
+     * those that we tried before, and download the remaining ones. Stop
+     * when there are no new URLs anymore. */
     do {
-      if (aw != null) {
-        urls.addAll(aw.getMissingDescriptorUrls());
+
+      /* Compile list of URLs to download in this iteration. */
+      urls.clear();
+      for (Map.Entry<String, String> e :
+          this.missingDescriptors.entrySet()) {
+        if (e.getValue().equals("NA")) {
+          String[] parts = e.getKey().split(",");
+          if (parts[0].equals("consensus") &&
+              this.downloadCurrentConsensus &&
+              this.currentValidAfter.equals(parts[1])) {
+            urls.add("/tor/status-vote/current/consensus");
+          } else if (parts[0].equals("vote") &&
+              this.downloadCurrentVotes &&
+              this.currentValidAfter.equals(parts[1])) {
+            urls.add("/tor/status-vote/current/" + parts[2]);
+          } else if (parts[0].equals("server") &&
+              (this.downloadAllServerDescriptors ||
+              (this.downloadDescriptorsForRelays != null &&
+              this.downloadDescriptorsForRelays.contains(parts[2].
+              toUpperCase()))) &&
+              this.descriptorCutOff.compareTo(parts[1]) <= 0) {
+            urls.add("/tor/server/d/" + parts[3]);
+          } else if (parts[0].equals("extra") &&
+              (this.downloadAllExtraInfos ||
+              (this.downloadDescriptorsForRelays != null &&
+              this.downloadDescriptorsForRelays.contains(parts[2].
+              toUpperCase()))) &&
+              this.descriptorCutOff.compareTo(parts[1]) <= 0) {
+            urls.add("/tor/extra/d/" + parts[3]);
+          }
+        }
       }
       urls.removeAll(downloaded);
-      SortedSet<String> sortedAuthorities =
-          new TreeSet<String>(remainingAuthorities);
-      SortedSet<String> sortedUrls = new TreeSet<String>(urls);
+
+      /* Log what we're downloading. */
+      StringBuilder sb = new StringBuilder("Downloading " + urls.size()
+          + " descriptors:");
+      for (String url : urls) {
+        sb.append(url + "\n");
+      }
+      this.logger.fine(sb.toString());
+
+      /* We are trying to download these descriptors from each directory
+       * source one after the other until we got it from one. For each
+       * directory source we are removing the URLs from urls and putting
+       * those the we want to retry into retryUrls. Once we are done, we
+       * move the URLs back to urls and try the next directory source. */
+      SortedSet<String> currentDirSources =
+          new TreeSet<String>(remainingDirSources);
       SortedSet<String> retryUrls = new TreeSet<String>();
-      while (!sortedAuthorities.isEmpty() && !sortedUrls.isEmpty()) {
-        String authority = sortedAuthorities.first();
-        String url = sortedUrls.first();
+      while (!currentDirSources.isEmpty() && !urls.isEmpty()) {
+        String authority = currentDirSources.first();
+        String url = urls.first();
         try {
           URL u = new URL("http://"; + authority + url);
           HttpURLConnection huc =
@@ -40,7 +399,7 @@ public class RelayDescriptorDownloader {
           huc.setRequestMethod("GET");
           huc.connect();
           int response = huc.getResponseCode();
-          logger.fine("Downloading http://"; + authority + url + " -> "
+          logger.finer("Downloading http://"; + authority + url + " -> "
               + response);
           if (response == 200) {
             BufferedInputStream in = new BufferedInputStream(
@@ -49,65 +408,25 @@ public class RelayDescriptorDownloader {
             int len;
             byte[] data = new byte[1024];
             while ((len = in.read(data, 0, 1024)) >= 0) {
-              // we need to write the result to a byte array in order
-              // to get a sane digest; otherwise, descriptors with
-              // non-ASCII chars lead to different digests.
               baos.write(data, 0, len);
             }
             in.close();
-            String digest = null;
             byte[] allData = baos.toByteArray();
-            int beforeSig = new String(allData).indexOf(
-                "\nrouter-signature\n")
-                + "\nrouter-signature\n".length();
-            byte[] noSig = new byte[beforeSig];
-            System.arraycopy(allData, 0, noSig, 0, beforeSig);
-            digest = DigestUtils.shaHex(noSig);
-            // TODO UTF-8 may be wrong, but we don't care about the fields
-            // containing non-ASCII
-            // String result = new String(allData, "UTF-8"); TODO
-            boolean verified = false;
-            if (url.contains("/tor/server/d/") ||
-                url.contains("/tor/extra/d/")) {
-              if (url.endsWith(digest)) {
-                verified = true;
-              } else {
-                logger.warning("Downloaded descriptor digest (" + digest
-                    + " doesn't match what we asked for (" + url + ")! "
-                    + "Retrying.");
-                retryUrls.add(url);
-              }
-            } else {
-              verified = true;
-              // TODO verify downloaded consensuses and votes, too
-            }
-            if (verified) {
-              if (rdp != null) {
-                rdp.parse(allData);
-              }
-              if (aw != null) {
-                try {
-                  aw.store(allData);
-                } catch (Exception e) {
-                  e.printStackTrace();
-                  //TODO find better way to handle this
-                }
-              }
-            }
+            rdp.parse(allData);
           } else {
             retryUrls.add(url);
           }
-          sortedUrls.remove(url);
-          if (sortedUrls.isEmpty()) {
-            sortedAuthorities.remove(authority);
-            sortedUrls.addAll(retryUrls);
+          urls.remove(url);
+          if (urls.isEmpty()) {
+            currentDirSources.remove(authority);
+            urls.addAll(retryUrls);
             retryUrls.clear();
           }
         } catch (IOException e) {
-          remainingAuthorities.remove(authority);
-          sortedAuthorities.remove(authority);
-          if (!remainingAuthorities.isEmpty()) {
-            logger.log(Level.INFO, "Failed downloading from "
+          remainingDirSources.remove(authority);
+          currentDirSources.remove(authority);
+          if (!remainingDirSources.isEmpty()) {
+            logger.log(Level.FINE, "Failed downloading from "
                 + authority + "!", e);
           } else {
             logger.log(Level.WARNING, "Failed downloading from "
@@ -119,5 +438,24 @@ public class RelayDescriptorDownloader {
       downloaded.addAll(urls);
     } while (!urls.isEmpty());
   }
-}
 
+  public void writeFile() {
+    try {
+      this.logger.fine("Writing file "
+          + this.missingDescriptorsFile.getAbsolutePath() + "...");
+      this.missingDescriptorsFile.getParentFile().mkdirs();
+      BufferedWriter bw = new BufferedWriter(new FileWriter(
+          this.missingDescriptorsFile));
+      for (Map.Entry<String, String> e :
+          this.missingDescriptors.entrySet()) {
+        bw.write(e.getKey() + "," + e.getValue() + "\n");
+      }
+      bw.close();
+      this.logger.fine("Finished writing file "
+          + this.missingDescriptorsFile.getAbsolutePath() + ".");
+    } catch (IOException e) {
+      this.logger.log(Level.WARNING, "Failed writing "
+          + this.missingDescriptorsFile.getAbsolutePath() + "!", e);
+    }
+  }
+}
diff --git a/src/RelayDescriptorParser.java b/src/RelayDescriptorParser.java
index 81661ef..b7af740 100644
--- a/src/RelayDescriptorParser.java
+++ b/src/RelayDescriptorParser.java
@@ -6,252 +6,304 @@ import org.apache.commons.codec.digest.*;
 import org.apache.commons.codec.binary.*;
 
 /**
- * Parse the contents of a network status consensus and pass on the
- * relevant contents to the stats file handlers.
+ * Parses relay descriptors including network status consensuses and
+ * votes, server and extra-info descriptors, and passes the results to the
+ * stats handlers, to the archive writer, or to the relay descriptor
+ * downloader.
  */
 public class RelayDescriptorParser {
-  private File relayDescriptorParseHistoryFile;
-  private SortedMap<String, String> lastParsedExtraInfos;
-  private String lastParsedConsensus;
-  private boolean relayDescriptorParseHistoryModified = false;
+
+  /**
+   * Stats file handler that accepts parse results for directory request
+   * statistics.
+   */
   private DirreqStatsFileHandler dsfh;
+
+  /**
+   * Stats file handler that accepts parse results for consensus
+   * statistics.
+   */
   private ConsensusStatsFileHandler csfh;
+
+  /**
+   * Stats file handler that accepts parse results for bridge statistics.
+   */
   private BridgeStatsFileHandler bsfh;
+
+  /**
+   * Stats file handler that accepts parse results for server descriptor
+   * statistics.
+   */
   private ServerDescriptorStatsFileHandler sdsfh;
+
+  /**
+   * File writer that writes descriptor contents to files in a
+   * directory-archive directory structure.
+   */
+  private ArchiveWriter aw;
+
+  /**
+   * Missing descriptor downloader that uses the parse results to learn
+   * which descriptors we are missing and want to download.
+   */
+  private RelayDescriptorDownloader rdd;
+
+  /**
+   * Countries that we care about for directory request and bridge
+   * statistics.
+   */
   private SortedSet<String> countries;
+
+  /**
+   * Directories that we care about for directory request statistics.
+   */
   private SortedSet<String> directories;
+
+  /**
+   * Logger for this class.
+   */
   private Logger logger;
+
+  /**
+   * Initializes this class.
+   */
   public RelayDescriptorParser(ConsensusStatsFileHandler csfh,
       BridgeStatsFileHandler bsfh, DirreqStatsFileHandler dsfh,
-      ServerDescriptorStatsFileHandler sdsfh, SortedSet<String> countries,
-      SortedSet<String> directories) {
-    this.relayDescriptorParseHistoryFile = new File(
-        "stats/relay-descriptor-parse-history");
+      ServerDescriptorStatsFileHandler sdsfh, ArchiveWriter aw,
+      SortedSet<String> countries, SortedSet<String> directories) {
     this.csfh = csfh;
     this.bsfh = bsfh;
     this.dsfh = dsfh;
     this.sdsfh = sdsfh;
+    this.aw = aw;
     this.countries = countries;
     this.directories = directories;
-    this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
-    this.lastParsedConsensus = null;
-    this.lastParsedExtraInfos = new TreeMap<String, String>();
-    if (this.relayDescriptorParseHistoryFile.exists()) {
-      this.logger.fine("Reading file "
-          + this.relayDescriptorParseHistoryFile.getAbsolutePath()
-          + "...");
-      try {
-        BufferedReader br = new BufferedReader(new FileReader(
-            this.relayDescriptorParseHistoryFile));
-        String line = null;
+  }
+
+  public void setRelayDescriptorDownloader(
+      RelayDescriptorDownloader rdd) {
+    this.rdd = rdd;
+  }
+
+  public void parse(byte[] data) {
+    try {
+      /* Convert descriptor to ASCII for parsing. This means we'll lose
+       * the non-ASCII chars, but we don't care about them for parsing
+       * anyway. */
+      BufferedReader br = new BufferedReader(new StringReader(new String(
+          data, "US-ASCII")));
+      String line = br.readLine();
+      if (line == null) {
+        this.logger.fine("We were given an empty descriptor for "
+            + "parsing. Ignoring.");
+        return;
+      }
+      SimpleDateFormat parseFormat =
+          new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+      parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+      if (line.equals("network-status-version 3")) {
+        // TODO when parsing the current consensus, check the fresh-until
+        // time to see when we switch from hourly to half-hourly
+        // consensuses
+        boolean isConsensus = true;
+        int exit = 0, fast = 0, guard = 0, running = 0, stable = 0;
+        String validAfterTime = null, descriptorIdentity = null;
+        StringBuilder descriptorIdentities = new StringBuilder();
+        String fingerprint = null;
+        long validAfter = -1L;
+        SortedSet<String> dirSources = new TreeSet<String>();
+        SortedSet<String> serverDescriptors = new TreeSet<String>();
+        SortedSet<String> hashedRelayIdentities = new TreeSet<String>();
         while ((line = br.readLine()) != null) {
-          if (line.startsWith("consensus")) {
-            this.lastParsedConsensus = line.split(",")[2];
-          } else if (line.startsWith("extrainfo")) {
-            this.lastParsedExtraInfos.put(line.split(",")[1],
-                line.split(",")[2]);
+          if (line.equals("vote-status vote")) {
+            isConsensus = false;
+          } else if (line.startsWith("valid-after ")) {
+            validAfterTime = line.substring("valid-after ".length());
+            validAfter = parseFormat.parse(validAfterTime).getTime();
+          } else if (line.startsWith("dir-source ")) {
+            dirSources.add(line.split(" ")[2]);
+          } else if (line.startsWith("fingerprint ")) {
+            fingerprint = line.split(" ")[1];
+          } else if (line.startsWith("r ")) {
+            String publishedTime = line.split(" ")[4] + " "
+                + line.split(" ")[5];
+            String relayIdentity = Hex.encodeHexString(
+                Base64.decodeBase64(line.split(" ")[2] + "=")).
+                toLowerCase();
+            String serverDesc = Hex.encodeHexString(Base64.decodeBase64(
+                line.split(" ")[3] + "=")).toLowerCase();
+            serverDescriptors.add(publishedTime + "," + relayIdentity
+                + "," + serverDesc);
+            hashedRelayIdentities.add(DigestUtils.shaHex(
+                Base64.decodeBase64(relayIdentity + "=")).toUpperCase());
+            descriptorIdentity = line.split(" ")[3];
+          } else if (line.startsWith("s ")) {
+            if (line.contains(" Running")) {
+              exit += line.contains(" Exit") ? 1 : 0;
+              fast += line.contains(" Fast") ? 1 : 0;
+              guard += line.contains(" Guard") ? 1 : 0;
+              stable += line.contains(" Stable") ? 1 : 0;
+              running++;
+              descriptorIdentities.append("," + descriptorIdentity);
+            }
           }
         }
-        br.close();
-        this.logger.fine("Finished reading file "
-            + this.relayDescriptorParseHistoryFile.getAbsolutePath()
-            + ".");
-      } catch (IOException e) {
-        this.logger.log(Level.WARNING, "Failed reading file "
-            + this.relayDescriptorParseHistoryFile.getAbsolutePath()
-            + "!", e);
-      }
-    }
-  }
-  public void parse(byte[] data) throws IOException {
-    BufferedReader br = new BufferedReader(new StringReader(new String(
-        data, "US-ASCII")));
-    String line = br.readLine();
-    if (line == null) {
-      this.logger.warning("Parsing empty file?");
-      return;
-    }
-    if (line.equals("network-status-version 3")) {
-      int exit = 0, fast = 0, guard = 0, running = 0, stable = 0;
-      String validAfter = null, rLine = null;
-      StringBuilder descriptorIdentities = new StringBuilder();
-      while ((line = br.readLine()) != null) {
-        if (line.startsWith("valid-after ")) {
-          validAfter = line.substring("valid-after ".length());
-          if (this.lastParsedConsensus == null ||
-              validAfter.compareTo(this.lastParsedConsensus) > 0) {
-            this.lastParsedConsensus = validAfter;
-            relayDescriptorParseHistoryModified = true;
-          }
-        } else if (line.equals("vote-status vote")) {
-          return;
-        } else if (line.startsWith("r ")) {
+        if (isConsensus) {
           if (this.bsfh != null) {
-            String hashedRelay = DigestUtils.shaHex(Base64.decodeBase64(
-                line.split(" ")[2] + "=")).toUpperCase();
-            this.bsfh.addHashedRelay(hashedRelay);
+            for (String hashedRelayIdentity : hashedRelayIdentities) {
+              this.bsfh.addHashedRelay(hashedRelayIdentity);
+            }    
+          }
+          if (this.csfh != null) {
+            this.csfh.addConsensusResults(validAfterTime, exit, fast,
+                guard, running, stable);
           }
-          rLine = line;
-        } else if (line.startsWith("s ")) {
-          if (line.contains(" Running")) {
-            exit += line.contains(" Exit") ? 1 : 0;
-            fast += line.contains(" Fast") ? 1 : 0;
-            guard += line.contains(" Guard") ? 1 : 0;
-            stable += line.contains(" Stable") ? 1 : 0;
-            running++;
-            descriptorIdentities.append("," + rLine.split(" ")[3]);
+          if (this.sdsfh != null) {
+            this.sdsfh.addConsensus(validAfterTime,
+                descriptorIdentities.toString().substring(1));
+          }
+          if (this.rdd != null) {
+            this.rdd.haveParsedConsensus(validAfterTime, dirSources,
+                serverDescriptors);
+          }
+          if (this.aw != null) {
+            this.aw.storeConsensus(data, validAfter);
+          }
+        } else {
+          if (this.rdd != null) {
+            this.rdd.haveParsedVote(validAfterTime, fingerprint,
+                serverDescriptors);
+          }
+          if (this.aw != null) {
+            String ascii = new String(data, "US-ASCII");
+            String startToken = "network-status-version ";
+            String sigToken = "directory-signature ";
+            int start = ascii.indexOf(startToken);
+            int sig = ascii.indexOf(sigToken);
+            if (start >= 0 && sig >= 0 && sig > start) {
+              sig += sigToken.length();
+              byte[] forDigest = new byte[sig - start];
+              System.arraycopy(data, start, forDigest, 0, sig - start);
+              String digest = DigestUtils.shaHex(forDigest).toUpperCase();
+              this.aw.storeVote(data, validAfter,
+                  new ArrayList<String>(dirSources).get(0), digest);
+            }
           }
         }
-      }
-      if (this.csfh != null) {
-        this.csfh.addConsensusResults(validAfter, exit, fast, guard,
-          running, stable);
-      }
-      if (this.sdsfh != null) {
-        this.sdsfh.addConsensus(validAfter,
-            descriptorIdentities.toString().substring(1));
-      }
-    } else if (line.startsWith("router ")) {
-      String platformLine = null, publishedLine = null,
-          bandwidthLine = null;
-      while ((line = br.readLine()) != null) {
-        if (line.startsWith("platform ")) {
-          platformLine = line;
-        } else if (line.startsWith("published ")) {
-          publishedLine = line;
-        } else if (line.startsWith("bandwidth ")) {
-          bandwidthLine = line;
+      } else if (line.startsWith("router ")) {
+        String platformLine = null, publishedLine = null,
+            publishedTime = null, bandwidthLine = null,
+            extraInfoDigest = null, relayIdentifier = null;
+        long published = -1L;
+        while ((line = br.readLine()) != null) {
+          if (line.startsWith("platform ")) {
+            platformLine = line;
+          } else if (line.startsWith("published ")) {
+            publishedLine = line;
+            publishedTime = line.substring("published ".length());
+            published = parseFormat.parse(publishedTime).getTime();
+          } else if (line.startsWith("opt fingerprint") ||
+              line.startsWith("fingerprint")) {
+            relayIdentifier = line.substring(line.startsWith("opt ") ?
+                "opt fingerprint".length() : "fingerprint".length()).
+                replaceAll(" ", "").toLowerCase();
+          } else if (line.startsWith("bandwidth ")) {
+            bandwidthLine = line;
+          } else if (line.startsWith("opt extra-info-digest ") ||
+              line.startsWith("extra-info-digest ")) {
+            extraInfoDigest = line.startsWith("opt ") ?
+                line.split(" ")[2].toLowerCase() :
+                line.split(" ")[1].toLowerCase();
+          }
         }
-      }
-      String ascii = new String(data, "US-ASCII");
-      String startToken = "router ";
-      String sigToken = "\nrouter-signature\n";
-      int start = ascii.indexOf(startToken);
-      int sig = ascii.indexOf(sigToken) + sigToken.length();
-      if (start < 0 || sig < 0 || sig < start) {
-        this.logger.warning("Cannot determine descriptor digest! "
-            + "Skipping.");
-        return;
-      }
-      byte[] forDigest = new byte[sig - start];
-      System.arraycopy(data, start, forDigest, 0, sig - start);
-      String descriptorIdentity = Base64.encodeBase64String(
-          DigestUtils.sha(forDigest)).substring(0, 27);
-      if (this.sdsfh != null) {
-        this.sdsfh.addServerDescriptor(descriptorIdentity, platformLine,
-            publishedLine, bandwidthLine);
-      }
-    } else if (line.startsWith("extra-info ") && this.dsfh != null &&
-        directories.contains(line.split(" ")[2])) {
-      String dir = line.split(" ")[2];
-      String statsEnd = null, date = null, v3ips = null;
-      boolean skip = false;
-      while ((line = br.readLine()) != null) {
-        if (line.startsWith("dirreq-stats-end ")) {
-          statsEnd = line.split(" ")[1] + " " + line.split(" ")[2];
-          date = line.split(" ")[1];
-          // trusted had very strange dirreq-v3-shares here...
-          skip = dir.equals("8522EB98C91496E80EC238E732594D1509158E77")
-              && (date.equals("2009-09-10") || date.equals("2009-09-11"));
-        } else if (line.startsWith("dirreq-v3-reqs ")
-            && line.length() > "dirreq-v3-reqs ".length()) {
-          v3ips = line.split(" ")[1];
-        } else if (line.startsWith("dirreq-v3-share ")
-            && v3ips != null && !skip) {
-          Map<String, String> obs = new HashMap<String, String>();
-          String[] parts = v3ips.split(",");
-          for (String p : parts) {
-            for (String c : this.countries) {
-              if (p.startsWith(c)) {
-                obs.put(c, p.substring(3));
+        String ascii = new String(data, "US-ASCII");
+        String startToken = "router ";
+        String sigToken = "\nrouter-signature\n";
+        int start = ascii.indexOf(startToken);
+        int sig = ascii.indexOf(sigToken) + sigToken.length();
+        String digest = null, descriptorIdentity = null;
+        if (start >= 0 || sig >= 0 || sig > start) {
+          byte[] forDigest = new byte[sig - start];
+          System.arraycopy(data, start, forDigest, 0, sig - start);
+          descriptorIdentity = Base64.encodeBase64String(
+              DigestUtils.sha(forDigest)).substring(0, 27);
+          digest = DigestUtils.shaHex(forDigest);
+        }
+        if (this.aw != null && digest != null) {
+          this.aw.storeServerDescriptor(data, digest, published);
+        }
+        if (this.rdd != null && digest != null) {
+          this.rdd.haveParsedServerDescriptor(publishedTime,
+              relayIdentifier, digest, extraInfoDigest);
+        }
+        if (this.sdsfh != null && descriptorIdentity != null) {
+          this.sdsfh.addServerDescriptor(descriptorIdentity, platformLine,
+              publishedLine, bandwidthLine);
+        }
+      } else if (line.startsWith("extra-info ")) {
+        String publishedTime = null, relayIdentifier = line.split(" ")[2];
+        long published = -1L;
+        String dir = line.split(" ")[2];
+        String date = null, v3Reqs = null;
+        boolean skip = false;
+        while ((line = br.readLine()) != null) {
+          if (line.startsWith("published ")) {
+            publishedTime = line.substring("published ".length());
+            published = parseFormat.parse(publishedTime).getTime();
+          } else if (line.startsWith("dirreq-stats-end ")) {
+            date = line.split(" ")[1];
+            // trusted had very strange dirreq-v3-shares here...
+            // TODO don't check that here, but in DirreqStatsFileHandler
+            skip = dir.equals("8522EB98C91496E80EC238E732594D1509158E77")
+                && (date.equals("2009-09-10") ||
+                    date.equals("2009-09-11"));
+          } else if (line.startsWith("dirreq-v3-reqs ")
+              && line.length() > "dirreq-v3-reqs ".length()) {
+            v3Reqs = line.split(" ")[1];
+          } else if (line.startsWith("dirreq-v3-share ")
+              && v3Reqs != null && !skip) {
+            Map<String, String> obs = new HashMap<String, String>();
+            String[] parts = v3Reqs.split(",");
+            for (String p : parts) {
+              for (String c : this.countries) {
+                if (p.startsWith(c)) {
+                  obs.put(c, p.substring(3));
+                }
               }
             }
-          }
-          String share = line.substring("dirreq-v3-share ".length(),
-              line.length() - 1);
-          this.dsfh.addObs(dir, date, obs, share);
-          if (!this.lastParsedExtraInfos.containsKey(dir) ||
-              statsEnd.compareTo(
-              this.lastParsedExtraInfos.get(dir)) > 0) {
-            this.lastParsedExtraInfos.put(dir, statsEnd);
-            relayDescriptorParseHistoryModified = true;
+            String share = line.substring("dirreq-v3-share ".length(),
+                line.length() - 1);
+            if (this.dsfh != null &&
+                directories.contains(relayIdentifier)) {
+              this.dsfh.addObs(dir, date, obs, share);
+            }
           }
         }
-      }
-    }
-  }
-  /**
-   * Returns the URLs of current descriptors that we are missing,
-   * including the current consensus and a few extra-info descriptors.
-   */
-  public Set<String> getMissingDescriptorUrls() {
-    Set<String> urls = new HashSet<String>();
-    // We might be missing the current consensus for either consensus
-    // stats or bridge stats; we remember ourselves which consensus we
-    // parsed before (most likely from parsing cached-consensus) and can
-    // decide whether we want a more current one
-    SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH");
-    format.setTimeZone(TimeZone.getTimeZone("UTC"));
-    String currentConsensus = format.format(new Date())
-        + ":00:00";
-    if (currentConsensus.equals(this.lastParsedConsensus)) {
-      urls.add("/tor/status-vote/current/consensus");
-    }
-    // We might be missing extra-info descriptors for dirreq stats for
-    // the directories we care about; we are happy with previous dirreq
-    // stats until they are more than 36 hours old (24 hours for the
-    // next stats period to end plus 12 hours for publishing a new
-    // descriptor)
-    format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-    format.setTimeZone(TimeZone.getTimeZone("UTC"));
-    long now = System.currentTimeMillis();
-    for (String directory : this.directories) {
-      if (!this.lastParsedExtraInfos.containsKey(directory)) {
-        urls.add("/tor/extra/fp/" + directory);
-      } else {
-        try {
-          long statsEnd = format.parse(this.lastParsedExtraInfos.get(
-              directory)).getTime();
-          if (statsEnd + 36L * 60L * 60L * 1000L < now) {
-            urls.add("/tor/extra/fp/" + directory);
-          }
-        } catch (ParseException e) {
-          this.logger.log(Level.WARNING, "Failed parsing timestamp in "
-              + this.relayDescriptorParseHistoryFile.getAbsolutePath()
-              + "!", e);
+        String ascii = new String(data, "US-ASCII");
+        String startToken = "extra-info ";
+        String sigToken = "\nrouter-signature\n";
+        String digest = null;
+        int start = ascii.indexOf(startToken);
+        int sig = ascii.indexOf(sigToken) + sigToken.length();
+        if (start >= 0 || sig >= 0 || sig > start) {
+          byte[] forDigest = new byte[sig - start];
+          System.arraycopy(data, start, forDigest, 0, sig - start);
+          digest = DigestUtils.shaHex(forDigest);
         }
-      }
-    }
-    return urls;
-  }
-  public void writeFile() {
-    if (this.relayDescriptorParseHistoryModified) {
-      try {
-        this.logger.fine("Writing file "
-            + this.relayDescriptorParseHistoryFile.getAbsolutePath()
-            + "...");
-        this.relayDescriptorParseHistoryFile.getParentFile().mkdirs();
-        BufferedWriter bw = new BufferedWriter(new FileWriter(
-            this.relayDescriptorParseHistoryFile));
-        bw.write("type,source,published\n");
-        if (this.lastParsedConsensus != null) {
-          bw.write("consensus,NA," + this.lastParsedConsensus + "\n");
+        if (this.aw != null && digest != null) {
+          this.aw.storeExtraInfoDescriptor(data, digest, published);
         }
-        for (Map.Entry<String, String> e :
-            this.lastParsedExtraInfos.entrySet()) {
-          bw.write("extrainfo," + e.getKey() + "," + e.getValue()
-              + "\n");
+        if (this.rdd != null && digest != null) {
+          this.rdd.haveParsedExtraInfoDescriptor(publishedTime,
+              relayIdentifier.toLowerCase(), digest);
         }
-        bw.close();
-        this.logger.fine("Finished writing file "
-            + this.relayDescriptorParseHistoryFile.getAbsolutePath()
-            + ".");
-      } catch (IOException e) {
-        this.logger.log(Level.WARNING, "Failed writing "
-            + this.relayDescriptorParseHistoryFile.getAbsolutePath()
-            + "!", e);
       }
+    } catch (IOException e) {
+      this.logger.log(Level.WARNING, "Could not parse descriptor. "
+          + "Skipping.", e);
+    } catch (ParseException e) {
+      this.logger.log(Level.WARNING, "Could not parse descriptor. "
+          + "Skipping.", e);
     }
   }
 }
-
-- 
1.6.5