[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-db/master] Download all server/extra-info descriptors once per day.



commit be6e7c3d4f9f9acdf5e2f459453c23a230c129d0
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Fri Apr 1 15:43:38 2011 +0200

    Download all server/extra-info descriptors once per day.
    
    The relay descriptor downloader so far only downloads server and
    extra-info descriptors that are referenced in other directory documents
    and that we are missing.
    
    With this patch we also download /tor/server/all and /tor/extra/all from
    the directories once per day to learn about server and extra-info
    descriptors that are not referenced anywhere.
    
    This patch further requests the compressed versions of descriptors by
    adding ".z" to the URLs in order to save bandwidth.
    
    This patch is a 50% rewrite of the relay descriptor downloader which
    became too complex over time.
---
 config.template                                    |   27 +-
 src/org/torproject/ernie/db/Configuration.java     |   56 ++-
 src/org/torproject/ernie/db/Main.java              |   15 +-
 .../ernie/db/RelayDescriptorDownloader.java        |  857 ++++++++++++--------
 4 files changed, 600 insertions(+), 355 deletions(-)

diff --git a/config.template b/config.template
index 47a8c8d..d744f25 100644
--- a/config.template
+++ b/config.template
@@ -24,7 +24,32 @@
 #
 ## Comma separated list of directory authority addresses (IP[:port]) to
 ## download missing relay descriptors from
-#DownloadFromDirectoryAuthorities 86.59.21.38,194.109.206.212,212.112.245.170
+#DownloadFromDirectoryAuthorities 86.59.21.38,216.224.124.114:9030,213.115.239.118:443,193.23.244.244,208.83.223.34:443,128.31.0.34:9131,194.109.206.212,212.112.245.170
+#
+## Download the current consensus (only if DownloadRelayDescriptors is 1)
+#DownloadCurrentConsensus 1
+#
+## Download current votes (only if DownloadRelayDescriptors is 1)
+#DownloadCurrentVotes 1
+#
+## Download missing server descriptors (only if DownloadRelayDescriptors
+## is 1)
+#DownloadMissingServerDescriptors 1
+#
+## Download missing extra-info descriptors (only if
+## DownloadRelayDescriptors is 1)
+#DownloadMissingExtraInfoDescriptors 1
+#
+## Download all server descriptors from the directory authorities at most
+## once a day (only if DownloadRelayDescriptors is 1)
+#DownloadAllServerDescriptors 0
+#
+## Download all extra-info descriptors from the directory authorities at
+## most once a day (only if DownloadRelayDescriptors is 1)
+#DownloadAllExtraInfoDescriptors 0
+#
+## Compress relay descriptors downloads by adding .z to the URLs
+#CompressRelayDescriptorDownloads 0
 #
 ## Import bridge snapshots from disk, if available
 #ImportBridgeSnapshots 0
diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java
index ae24175..b4e6f8b 100644
--- a/src/org/torproject/ernie/db/Configuration.java
+++ b/src/org/torproject/ernie/db/Configuration.java
@@ -30,8 +30,17 @@ public class Configuration {
   private boolean importBridgeSnapshots = false;
   private String bridgeSnapshotsDirectory = "bridge-directories/";
   private boolean downloadRelayDescriptors = false;
-  private List<String> downloadFromDirectoryAuthorities = Arrays.asList(
-      "86.59.21.38,194.109.206.212,80.190.246.100:8180".split(","));
+  private List<String> downloadFromDirectoryAuthorities = Arrays.asList((
+      "86.59.21.38,216.224.124.114:9030,213.115.239.118:443,"
+      + "193.23.244.244,208.83.223.34:443,128.31.0.34:9131,"
+      + "194.109.206.212,212.112.245.170").split(","));
+  private boolean downloadCurrentConsensus = true;
+  private boolean downloadCurrentVotes = true;
+  private boolean downloadMissingServerDescriptors = true;
+  private boolean downloadMissingExtraInfoDescriptors = true;
+  private boolean downloadAllServerDescriptors = false;
+  private boolean downloadAllExtraInfoDescriptors = false;
+  private boolean compressRelayDescriptorDownloads;
   private boolean downloadGetTorStats = false;
   private String getTorStatsUrl = "http://gettor.torproject.org:8080/";
       + "~gettor/gettor_stats.txt";
@@ -117,6 +126,28 @@ public class Configuration {
             new URL("http://"; + dir + "/");
             this.downloadFromDirectoryAuthorities.add(dir);
           }
+        } else if (line.startsWith("DownloadCurrentConsensus")) {
+          this.downloadCurrentConsensus = Integer.parseInt(
+              line.split(" ")[1]) != 0;
+        } else if (line.startsWith("DownloadCurrentVotes")) {
+          this.downloadCurrentVotes = Integer.parseInt(
+              line.split(" ")[1]) != 0;
+        } else if (line.startsWith("DownloadMissingServerDescriptors")) {
+          this.downloadMissingServerDescriptors = Integer.parseInt(
+              line.split(" ")[1]) != 0;
+        } else if (line.startsWith(
+            "DownloadMissingExtraInfoDescriptors")) {
+          this.downloadMissingExtraInfoDescriptors = Integer.parseInt(
+              line.split(" ")[1]) != 0;
+        } else if (line.startsWith("DownloadAllServerDescriptors")) {
+          this.downloadAllServerDescriptors = Integer.parseInt(
+              line.split(" ")[1]) != 0;
+        } else if (line.startsWith("DownloadAllExtraInfoDescriptors")) {
+          this.downloadAllExtraInfoDescriptors = Integer.parseInt(
+              line.split(" ")[1]) != 0;
+        } else if (line.startsWith("CompressRelayDescriptorDownloads")) {
+          this.compressRelayDescriptorDownloads = Integer.parseInt(
+              line.split(" ")[1]) != 0;
         } else if (line.startsWith("DownloadGetTorStats")) {
           this.downloadGetTorStats = Integer.parseInt(
               line.split(" ")[1]) != 0;
@@ -244,6 +275,27 @@ public class Configuration {
   public List<String> getDownloadFromDirectoryAuthorities() {
     return this.downloadFromDirectoryAuthorities;
   }
+  public boolean getDownloadCurrentConsensus() {
+    return this.downloadCurrentConsensus;
+  }
+  public boolean getDownloadCurrentVotes() {
+    return this.downloadCurrentVotes;
+  }
+  public boolean getDownloadMissingServerDescriptors() {
+    return this.downloadMissingServerDescriptors;
+  }
+  public boolean getDownloadMissingExtraInfoDescriptors() {
+    return this.downloadMissingExtraInfoDescriptors;
+  }
+  public boolean getDownloadAllServerDescriptors() {
+    return this.downloadAllServerDescriptors;
+  }
+  public boolean getDownloadAllExtraInfoDescriptors() {
+    return this.downloadAllExtraInfoDescriptors;
+  }
+  public boolean getCompressRelayDescriptorDownloads() {
+    return this.compressRelayDescriptorDownloads;
+  }
   public boolean getDownloadGetTorStats() {
     return this.downloadGetTorStats;
   }
diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java
index 50c06bb..5a06900 100644
--- a/src/org/torproject/ernie/db/Main.java
+++ b/src/org/torproject/ernie/db/Main.java
@@ -49,13 +49,14 @@ public class Main {
       if (config.getDownloadRelayDescriptors()) {
         List<String> dirSources =
             config.getDownloadFromDirectoryAuthorities();
-        boolean downloadCurrentConsensus = aw != null;
-        boolean downloadCurrentVotes = aw != null;
-        boolean downloadAllServerDescriptors = aw != null;
-        boolean downloadAllExtraInfos = aw != null;
         rdd = new RelayDescriptorDownloader(rdp, dirSources,
-            downloadCurrentConsensus, downloadCurrentVotes,
-            downloadAllServerDescriptors, downloadAllExtraInfos);
+            config.getDownloadCurrentConsensus(),
+            config.getDownloadCurrentVotes(),
+            config.getDownloadMissingServerDescriptors(),
+            config.getDownloadMissingExtraInfoDescriptors(),
+            config.getDownloadAllServerDescriptors(),
+            config.getDownloadAllExtraInfoDescriptors(),
+            config.getCompressRelayDescriptorDownloads());
         rdp.setRelayDescriptorDownloader(rdd);
       }
       if (config.getImportCachedRelayDescriptors()) {
@@ -77,7 +78,7 @@ public class Main {
         }
       }
       if (rdd != null) {
-        rdd.downloadMissingDescriptors();
+        rdd.downloadDescriptors();
         rdd.writeFile();
         rdd = null;
         if (aw != null) {
diff --git a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java
index 56d7e0d..ebbf761 100644
--- a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java
+++ b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java
@@ -7,27 +7,29 @@ import java.net.*;
 import java.text.*;
 import java.util.*;
 import java.util.logging.*;
+import java.util.zip.*;
 
 import org.apache.commons.codec.digest.*;
 import org.apache.commons.codec.binary.*;
 
 /**
- * Downloads missing relay descriptors from the directories via HTTP.
+ * Downloads relay descriptors from the directory authorities via HTTP.
  * Keeps a list of missing descriptors that gets updated by parse results
- * from <code>RelayDescriptorParser</code>. Only descriptors on that
- * missing list that we think might be available on the directories are
- * downloaded.
+ * from <code>RelayDescriptorParser</code> and downloads all missing
+ * descriptors that have been published in the last 24 hours. Also
+ * downloads all server and extra-info descriptors known to a directory
+ * authority at most once a day.
  */
 public class RelayDescriptorDownloader {
 
   /**
    * Text file containing the descriptors that we are missing and that we
-   * want to download in <code>downloadMissingDescriptors</code>.
-   * Lines are formatted as:
+   * want to download. Lines are formatted as:
+   *
    * - "consensus,<validafter>,<parsed>",
    * - "vote,<validafter>,<fingerprint>,<parsed>",
    * - "server,<published>,<relayid>,<descid>,<parsed>", or
-   * - "extra,<published>,<relayid>,<descid><parsed>".
+   * - "extra,<published>,<relayid>,<descid>,<parsed>".
    */
   private File missingDescriptorsFile;
 
@@ -35,25 +37,36 @@ public class RelayDescriptorDownloader {
    * Relay descriptors that we are missing and that we want to download
    * either in this execution or write to disk and try next time. Map keys
    * contain comma-separated values as in the missing descriptors files
-   * without the parsed column. Map values contain the parsed column.
+   * without the "parsed" column. Map values contain the "parsed" column.
    */
   private SortedMap<String, String> missingDescriptors;
 
   /**
-   * <code>RelayDescriptorParser</code> that we will hand over the
-   * downloaded descriptors for parsing.
+   * Text file containing the IP addresses (and Dir ports if not 80) of
+   * directory authorities and when we last downloaded all server and
+   * extra-info descriptors from them, so that we can avoid downloading
+   * them too often.
    */
-  private RelayDescriptorParser rdp;
+  private File lastDownloadedAllDescriptorsFile;
+
+  /**
+   * Map of directory authorities and when we last downloaded all server
+   * and extra-info descriptors from them. Map keys are IP addresses (and
+   * Dir ports if not 80), map values are timestamps.
+   */
+  private Map<String, String> lastDownloadedAllDescriptors;
 
   /**
-   * Directories that we will try to download missing descriptors from.
+   * <code>RelayDescriptorParser</code> that we will hand over the
+   * downloaded descriptors for parsing.
    */
-  private List<String> dirSources;
+  private RelayDescriptorParser rdp;
 
   /**
-   * Number of descriptors requested by directory to be included in logs.
+   * Directory authorities that we will try to download missing
+   * descriptors from.
    */
-  private Map<String, Integer> dirRequests;
+  private List<String> authorities;
 
   /**
    * Should we try to download the current consensus if we don't have it?
@@ -66,22 +79,41 @@ public class RelayDescriptorDownloader {
   private boolean downloadCurrentVotes;
 
   /**
-   * Should we try to download all missing server descriptors that have
+   * Should we try to download missing server descriptors that have been
+   * published within the past 24 hours?
+   */
+  private boolean downloadMissingServerDescriptors;
+
+  /**
+   * Should we try to download missing extra-info descriptors that have
    * been published within the past 24 hours?
    */
+  private boolean downloadMissingExtraInfos;
+
+  /**
+   * Should we try to download all server descriptors from the authorities
+   * once every 24 hours?
+   */
   private boolean downloadAllServerDescriptors;
 
   /**
-   * Should we try to download all missing extra-info descriptors that
-   * have been published within the past 24 hours?
+   * Should we try to download all extra-info descriptors from the
+   * authorities once every 24 hours?
    */
   private boolean downloadAllExtraInfos;
 
   /**
+   * Should we download zlib-compressed versions of descriptors by adding
+   * ".z" to URLs?
+   */
+  private boolean downloadCompressed;
+
+  /**
    * valid-after time that we expect the current consensus and votes to
    * have, formatted "yyyy-MM-dd HH:mm:ss". We only expect to find
-   * consensuses and votes with this valid-after time on the directories.
-   * This time is initialized as the beginning of the current hour.
+   * consensuses and votes with this valid-after time on the directory
+   * authorities. This time is initialized as the beginning of the current
+   * hour.
    */
   private String currentValidAfter;
 
@@ -93,77 +125,115 @@ public class RelayDescriptorDownloader {
   private String descriptorCutOff;
 
   /**
+   * Cut-off time for downloading all server and extra-info descriptors
+   * from the directory authorities, formatted "yyyy-MM-dd HH:mm:ss". This
+   * time is initialized as the current system time minus 23:30 hours.
+   */
+  private String downloadAllDescriptorsCutOff;
+
+  /**
+   * Directory authorities that we plan to download all server and
+   * extra-info descriptors from in this execution.
+   */
+  private Set<String> downloadAllDescriptorsFromAuthorities;
+
+  /**
    * Current timestamp that is written to the missing list for descriptors
-   * that we parsed in this execution. This timestamp is most useful for
-   * debugging purposes when looking at the missing list. For execution it
-   * only matters whether the parsed time is "NA" or has some other value.
+   * that we parsed in this execution and for authorities that we
+   * downloaded all server and extra-info descriptors from.
    */
-  private String parsedTimestampString;
+  private String currentTimestamp;
 
   /**
    * Logger for this class.
    */
   private Logger logger;
 
-  private StringBuilder dumpStats;
-  private int newMissingConsensuses = 0, newMissingVotes = 0,
+  /**
+   * Number of descriptors requested by directory authority to be included
+   * in logs.
+   */
+  private Map<String, Integer> requestsByAuthority;
+
+  /**
+   * Counters for descriptors that we had on the missing list at the
+   * beginning of the execution, that we added to the missing list,
+   * that we requested, and that we successfully downloaded in this
+   * execution.
+   */
+  private int oldMissingConsensuses = 0, oldMissingVotes = 0,
+      oldMissingServerDescriptors = 0, oldMissingExtraInfoDescriptors = 0,
+      newMissingConsensuses = 0, newMissingVotes = 0,
       newMissingServerDescriptors = 0, newMissingExtraInfoDescriptors = 0,
-      triedConsensuses = 0, triedVotes = 0, triedServerDescriptors = 0,
-      triedExtraInfoDescriptors = 0, downloadedConsensuses = 0,
-      downloadedVotes = 0, downloadedServerDescriptors = 0,
-      downloadedExtraInfoDescriptors = 0;
+      requestedConsensuses = 0, requestedVotes = 0,
+      requestedMissingServerDescriptors = 0,
+      requestedAllServerDescriptors = 0,
+      requestedMissingExtraInfoDescriptors = 0,
+      requestedAllExtraInfoDescriptors = 0, downloadedConsensuses = 0,
+      downloadedVotes = 0, downloadedMissingServerDescriptors = 0,
+      downloadedAllServerDescriptors = 0,
+      downloadedMissingExtraInfoDescriptors = 0,
+      downloadedAllExtraInfoDescriptors = 0;
+
   /**
    * Initializes this class, including reading in missing descriptors from
-   * <code>stats/missing-relay-descriptors</code>.
+   * <code>stats/missing-relay-descriptors</code> and the times when we
+   * last downloaded all server and extra-info descriptors from
+   * <code>stats/last-downloaded-all-descriptors</code>.
    */
   public RelayDescriptorDownloader(RelayDescriptorParser rdp,
-      List<String> dirSources, boolean downloadCurrentConsensus,
-      boolean downloadCurrentVotes, boolean downloadAllServerDescriptors,
-      boolean downloadAllExtraInfos) {
+      List<String> authorities, boolean downloadCurrentConsensus,
+      boolean downloadCurrentVotes,
+      boolean downloadMissingServerDescriptors,
+      boolean downloadMissingExtraInfos,
+      boolean downloadAllServerDescriptors, boolean downloadAllExtraInfos,
+      boolean downloadCompressed) {
 
     /* Memorize argument values. */
     this.rdp = rdp;
-    this.dirSources = dirSources;
+    this.authorities = new ArrayList<String>(authorities);
     this.downloadCurrentConsensus = downloadCurrentConsensus;
     this.downloadCurrentVotes = downloadCurrentVotes;
+    this.downloadMissingServerDescriptors =
+        downloadMissingServerDescriptors;
+    this.downloadMissingExtraInfos = downloadMissingExtraInfos;
     this.downloadAllServerDescriptors = downloadAllServerDescriptors;
     this.downloadAllExtraInfos = downloadAllExtraInfos;
+    this.downloadCompressed = downloadCompressed;
+
+    /* Shuffle list of authorities for better load balancing over time. */
+    Collections.shuffle(this.authorities);
 
     /* Initialize logger. */
-    this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
+    this.logger = Logger.getLogger(
+        RelayDescriptorDownloader.class.getName());
 
-    /* Prepare cut-off times and timestamp for missing descriptors
-     * list. */
+    /* Prepare cut-off times and timestamp for the missing descriptors
+     * list and the list of authorities to download all server and
+     * extra-info descriptors from. */
     SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
     format.setTimeZone(TimeZone.getTimeZone("UTC"));
     long now = System.currentTimeMillis();
     this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) *
         (60L * 60L * 1000L));
     this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L);
-    this.parsedTimestampString = format.format(now);
-
-    /* Initialize missing list and put current consensus on it if we want
-     * it. */
-    this.missingDescriptors = new TreeMap<String, String>();
-    if (this.downloadCurrentConsensus) {
-      this.missingDescriptors.put("consensus," + this.currentValidAfter,
-          "NA");
-    }
+    this.currentTimestamp = format.format(now);
+    this.downloadAllDescriptorsCutOff = format.format(now
+        - 23L * 60L * 60L * 1000L - 30L * 60L * 1000L);
 
     /* Read list of missing descriptors from disk and memorize those that
      * we are interested in and that are likely to be found on the
-     * directory servers. */
+     * directory authorities. */
+    this.missingDescriptors = new TreeMap<String, String>();
     this.missingDescriptorsFile = new File(
         "stats/missing-relay-descriptors");
-    int missingConsensuses = 0, missingVotes = 0,
-        missingServerDescriptors = 0, missingExtraInfoDescriptors = 0;
     if (this.missingDescriptorsFile.exists()) {
       try {
         this.logger.fine("Reading file "
             + this.missingDescriptorsFile.getAbsolutePath() + "...");
         BufferedReader br = new BufferedReader(new FileReader(
             this.missingDescriptorsFile));
-        String line = null;
+        String line;
         while ((line = br.readLine()) != null) {
           if (line.split(",").length > 2) {
             String published = line.split(",")[1];
@@ -172,17 +242,17 @@ public class RelayDescriptorDownloader {
                 this.currentValidAfter.equals(published)) ||
                 ((line.startsWith("server,") ||
                 line.startsWith("extra,")) &&
-                this.descriptorCutOff.compareTo(published) <= 0)) {
+                this.descriptorCutOff.compareTo(published) < 0)) {
               if (!line.endsWith("NA")) {
                 /* Not missing. */
               } else if (line.startsWith("consensus,")) {
-                missingConsensuses++;
+                oldMissingConsensuses++;
               } else if (line.startsWith("vote,")) {
-                missingVotes++;
+                oldMissingVotes++;
               } else if (line.startsWith("server,")) {
-                missingServerDescriptors++;
+                oldMissingServerDescriptors++;
               } else if (line.startsWith("extra,")) {
-                missingExtraInfoDescriptors++;
+                oldMissingExtraInfoDescriptors++;
               }
               int separateAt = line.lastIndexOf(",");
               this.missingDescriptors.put(line.substring(0,
@@ -205,86 +275,131 @@ public class RelayDescriptorDownloader {
       }
     }
 
-    dumpStats = new StringBuilder();
-    dumpStats.append("Finished downloading relay descriptors from the "
-        + "directory authorities:\nAt the beginning of this execution, "
-        + "we were missing " + missingConsensuses + " consensus(es), "
-        + missingVotes + " vote(s), " + missingServerDescriptors
-        + " server descriptor(s), and " + missingExtraInfoDescriptors
-        + " extra-info descriptor(s).\n");
-
-    dirRequests = new HashMap<String, Integer>();
-    for (String dirSource : dirSources) {
-      dirRequests.put(dirSource, 0);
+    /* Put the current consensus on the missing list, unless we already
+     * have it. */
+    String consensusKey = "consensus," + this.currentValidAfter;
+    if (!this.missingDescriptors.containsKey(consensusKey)) {
+      this.missingDescriptors.put(consensusKey, "NA");
+      this.newMissingConsensuses++;
+    }
+
+    /* Read list of directory authorities and when we last downloaded all
+     * server and extra-info descriptors from them. */
+    this.lastDownloadedAllDescriptors = new HashMap<String, String>();
+    this.lastDownloadedAllDescriptorsFile = new File(
+        "stats/last-downloaded-all-descriptors");
+    if (this.lastDownloadedAllDescriptorsFile.exists()) {
+      try {
+        this.logger.fine("Reading file "
+            + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+            + "...");
+        BufferedReader br = new BufferedReader(new FileReader(
+            this.lastDownloadedAllDescriptorsFile));
+        String line;
+        while ((line = br.readLine()) != null) {
+          if (line.split(",").length != 2) {
+            this.logger.fine("Invalid line '" + line + "' in "
+                + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+                + ". Ignoring.");
+          } else {
+            String[] parts = line.split(",");
+            String authority = parts[0];
+            String lastDownloaded = parts[1];
+            this.lastDownloadedAllDescriptors.put(authority,
+                lastDownloaded);
+          }
+        }
+        br.close();
+        this.logger.fine("Finished reading file "
+            + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+            + ".");
+      } catch (IOException e) {
+        this.logger.log(Level.WARNING, "Failed to read file "
+            + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+            + "! This means that we might download all server and "
+            + "extra-info descriptors more often than we should.", e);
+      }
+    }
+
+    /* Make a list of directory authorities that we want to download all
+     * server and extra-info descriptors from. */
+    this.downloadAllDescriptorsFromAuthorities = new HashSet<String>();
+    for (String authority : this.authorities) {
+      if (!this.lastDownloadedAllDescriptors.containsKey(authority) ||
+          this.lastDownloadedAllDescriptors.get(authority).compareTo(
+          this.downloadAllDescriptorsCutOff) < 0) {
+        this.downloadAllDescriptorsFromAuthorities.add(authority);
+      }
+    }
+
+    /* Prepare statistics on this execution. */
+    this.requestsByAuthority = new HashMap<String, Integer>();
+    for (String authority : this.authorities) {
+      this.requestsByAuthority.put(authority, 0);
     }
   }
 
   /**
    * We have parsed a consensus. Take this consensus off the missing list
-   * and add the votes created by the given <code>dirSources</code> and
-   * the <code>serverDescriptors</code> in the format
+   * and add the votes created by the given <code>authorities</code> and
+   * the <code>serverDescriptors</code> which are in the format
    * "<published>,<relayid>,<descid>" to that list.
    */
   public void haveParsedConsensus(String validAfter,
-      Set<String> dirSources, Set<String> serverDescriptors) {
+      Set<String> authorities, Set<String> serverDescriptors) {
 
     /* Mark consensus as parsed. */
     if (this.currentValidAfter.equals(validAfter)) {
       String consensusKey = "consensus," + validAfter;
-      this.missingDescriptors.put(consensusKey,
-          this.parsedTimestampString);
+      this.missingDescriptors.put(consensusKey, this.currentTimestamp);
 
       /* Add votes to missing list. */
-      if (this.downloadCurrentVotes) {
-        for (String dirSource : dirSources) {
-          String voteKey = "vote," + validAfter + "," + dirSource;
-          if (!this.missingDescriptors.containsKey(voteKey)) {
-            this.missingDescriptors.put(voteKey, "NA");
-            this.newMissingVotes++;
-          }
+      for (String authority : authorities) {
+        String voteKey = "vote," + validAfter + "," + authority;
+        if (!this.missingDescriptors.containsKey(voteKey)) {
+          this.missingDescriptors.put(voteKey, "NA");
+          this.newMissingVotes++;
         }
       }
     }
 
     /* Add server descriptors to missing list. */
-    if (this.downloadAllServerDescriptors) {
-      for (String serverDescriptor : serverDescriptors) {
-        String published = serverDescriptor.split(",")[0];
-        if (this.descriptorCutOff.compareTo(published) <= 0) {
-          String serverDescriptorKey = "server," + serverDescriptor;
-          if (!this.missingDescriptors.containsKey(
-              serverDescriptorKey)) {
-            this.missingDescriptors.put(serverDescriptorKey, "NA");
-            this.newMissingServerDescriptors++;
-          }
+    for (String serverDescriptor : serverDescriptors) {
+      String published = serverDescriptor.split(",")[0];
+      if (this.descriptorCutOff.compareTo(published) < 0) {
+        String serverDescriptorKey = "server," + serverDescriptor;
+        if (!this.missingDescriptors.containsKey(
+            serverDescriptorKey)) {
+          this.missingDescriptors.put(serverDescriptorKey, "NA");
+          this.newMissingServerDescriptors++;
         }
       }
     }
   }
 
   /**
-   * We have parsed a vote. Take this vote off the missing list.
+   * We have parsed a vote. Take this vote off the missing list and add
+   * the <code>serverDescriptors</code> which are in the format
+   * "<published>,<relayid>,<descid>" to that list.
    */
   public void haveParsedVote(String validAfter, String fingerprint,
       Set<String> serverDescriptors) {
 
-    /* Mark consensus as parsed. */
+    /* Mark vote as parsed. */
     if (this.currentValidAfter.equals(validAfter)) {
       String voteKey = "vote," + validAfter + "," + fingerprint;
-      this.missingDescriptors.put(voteKey, this.parsedTimestampString);
+      this.missingDescriptors.put(voteKey, this.currentTimestamp);
     }
 
     /* Add server descriptors to missing list. */
-    if (this.downloadAllServerDescriptors) {
-      for (String serverDescriptor : serverDescriptors) {
-        String published = serverDescriptor.split(",")[0];
-        if (this.descriptorCutOff.compareTo(published) < 0) {
-          String serverDescriptorKey = "server," + serverDescriptor;
-          if (!this.missingDescriptors.containsKey(
-              serverDescriptorKey)) {
-            this.missingDescriptors.put(serverDescriptorKey, "NA");
-            this.newMissingServerDescriptors++;
-          }
+    for (String serverDescriptor : serverDescriptors) {
+      String published = serverDescriptor.split(",")[0];
+      if (this.descriptorCutOff.compareTo(published) < 0) {
+        String serverDescriptorKey = "server," + serverDescriptor;
+        if (!this.missingDescriptors.containsKey(
+            serverDescriptorKey)) {
+          this.missingDescriptors.put(serverDescriptorKey, "NA");
+          this.newMissingServerDescriptors++;
         }
       }
     }
@@ -300,14 +415,14 @@ public class RelayDescriptorDownloader {
       String extraInfoDigest) {
 
     /* Mark server descriptor as parsed. */
-    if (this.descriptorCutOff.compareTo(published) <= 0) {
+    if (this.descriptorCutOff.compareTo(published) < 0) {
       String serverDescriptorKey = "server," + published + ","
           + relayIdentity + "," + serverDescriptorDigest;
       this.missingDescriptors.put(serverDescriptorKey,
-          this.parsedTimestampString);
+          this.currentTimestamp);
 
       /* Add extra-info descriptor to missing list. */
-      if (extraInfoDigest != null && this.downloadAllExtraInfos) {
+      if (extraInfoDigest != null) {
         String extraInfoKey = "extra," + published + ","
             + relayIdentity + "," + extraInfoDigest;
         if (!this.missingDescriptors.containsKey(extraInfoKey)) {
@@ -324,253 +439,261 @@ public class RelayDescriptorDownloader {
    */
   public void haveParsedExtraInfoDescriptor(String published,
       String relayIdentity, String extraInfoDigest) {
-    if (this.descriptorCutOff.compareTo(published) <= 0) {
+    if (this.descriptorCutOff.compareTo(published) < 0) {
       String extraInfoKey = "extra," + published + ","
           + relayIdentity + "," + extraInfoDigest;
-      this.missingDescriptors.put(extraInfoKey,
-          this.parsedTimestampString);
+      this.missingDescriptors.put(extraInfoKey, this.currentTimestamp);
     }
   }
 
   /**
    * Downloads missing descriptors that we think might still be available
-   * on the directories.
+   * on the directory authorities as well as all server and extra-info
+   * descriptors once per day.
    */
-  public void downloadMissingDescriptors() {
-
-    /* Update cut-off times to reflect that execution so far might have
-     * taken a few minutes and that some descriptors aren't available on
-     * the directories anymore. */
-    SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-    format.setTimeZone(TimeZone.getTimeZone("UTC"));
-    long now = System.currentTimeMillis();
-    this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) *
-        (60L * 60L * 1000L));
-    this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L);
+  public void downloadDescriptors() {
+
+    /* Download descriptors from authorities which are in random order, so
+     * that we distribute the load somewhat fairly over time. */
+    for (String authority : authorities) {
+
+      /* Make all requests to an authority in a single try block. If
+       * something goes wrong with this authority, we give up on all
+       * downloads and continue with the next authority. */
+      /* TODO Some authorities provide very little bandwidth and could
+       * slow down the entire download process. Ponder adding a timeout of
+       * 3 or 5 minutes per authority to avoid getting in the way of the
+       * next execution. */
+      try {
 
-    /* Remember which directories remain as source for downloading
-     * descriptors. */
-    List<String> remainingDirSources =
-        new ArrayList<String>(this.dirSources);
-
-    /* URLs of descriptors we want to download. */
-    SortedSet<String> urls = new TreeSet<String>();
-
-    /* Complete URLs of authorities and descriptors we have downloaded or
-     * tried to download. Ensures that we're not attempting to download
-     * the same thing from an authority more than once. There are edge
-     * cases when an authority returns a valid response to something we
-     * asked for, but which is not what we wanted (e.g. old consensus). */
-    SortedSet<String> downloaded = new TreeSet<String>();
-
-    /* We might need more than one iteration for downloading descriptors,
-     * because we might learn about descriptors while parsing those that
-     * we got. In every iteration, compile a new list of URLs, remove
-     * those that we tried before, and download the remaining ones. Stop
-     * when there are no new URLs anymore. */
-    while (true) {
-
-      /* Compile list of URLs to download in this iteration. */
-      urls.clear();
-      for (Map.Entry<String, String> e :
-          this.missingDescriptors.entrySet()) {
-        if (e.getValue().equals("NA")) {
-          String[] parts = e.getKey().split(",");
-          if (parts[0].equals("consensus") &&
-              this.downloadCurrentConsensus &&
-              this.currentValidAfter.equals(parts[1])) {
-            urls.add("/tor/status-vote/current/consensus");
-          } else if (parts[0].equals("vote") &&
-              this.downloadCurrentVotes &&
-              this.currentValidAfter.equals(parts[1])) {
-            urls.add("/tor/status-vote/current/" + parts[2]);
-          } else if (parts[0].equals("server") &&
-              this.downloadAllServerDescriptors &&
-              this.descriptorCutOff.compareTo(parts[1]) <= 0) {
-            urls.add("/tor/server/d/" + parts[3]);
-          } else if (parts[0].equals("extra") &&
-              this.downloadAllExtraInfos &&
-              this.descriptorCutOff.compareTo(parts[1]) <= 0) {
-            urls.add("/tor/extra/d/" + parts[3]);
+        /* Start with downloading the current consensus, unless we already
+         * have it. */
+        if (downloadCurrentConsensus) {
+          String consensusKey = "consensus," + this.currentValidAfter;
+          if (this.missingDescriptors.containsKey(consensusKey) &&
+              this.missingDescriptors.get(consensusKey).equals("NA")) {
+            this.requestedConsensuses++;
+            this.downloadedConsensuses +=
+                this.downloadResourceFromAuthority(authority,
+                "/tor/status-vote/current/consensus");
           }
         }
-      }
 
-      /* Stop if we don't have (new) URLs to download. */
-      if (urls.isEmpty()) {
-        break;
-      }
-
-      for (String url : urls) {
-        if (url.endsWith("consensus")) {
-          this.triedConsensuses++;
-        } else if (url.contains("status-vote")) {
-          this.triedVotes++;
-        } else if (url.contains("server")) {
-          this.triedServerDescriptors++;
-        } else if (url.contains("extra")) {
-          this.triedExtraInfoDescriptors++;
-        }
-      }
-
-      /* Log what we're downloading. */
-      StringBuilder sb = new StringBuilder("Downloading " + urls.size()
-          + " descriptors:");
-      for (String url : urls) {
-        sb.append("\n" + url);
-      }
-      this.logger.fine(sb.toString());
-
-      /* We are trying to download these descriptors from each directory
-       * source one after the other until we got it from one. For each
-       * directory source we are removing the URLs from urls and putting
-       * those the we want to retry into retryUrls. Once we are done, we
-       * move the URLs back to urls and try the next directory source. */
-      SortedSet<String> currentDirSources =
-          new TreeSet<String>(remainingDirSources);
-      SortedSet<String> retryUrls = new TreeSet<String>();
-      int numDownloaded = 0;
-      while (!currentDirSources.isEmpty() && !urls.isEmpty()) {
-        String authority = currentDirSources.first();
-        String url = urls.first();
-        urls.remove(url);
-        Set<String> requestedUrls = new HashSet<String>();
-        requestedUrls.add(url);
-        if (url.contains("server")) {
-          StringBuilder combinedUrl = new StringBuilder(url);
-          int descriptors = 1;
-          while (descriptors++ <= 96 && !urls.isEmpty() &&
-              urls.first().contains("server")) {
-            url = urls.first();
-            urls.remove(url);
-            requestedUrls.add(url);
-            combinedUrl.append("+" + url.substring("/tor/server/d/".
-                length()));
+        /* Next, try to download current votes that we're missing. */
+        if (downloadCurrentVotes) {
+          String voteKeyPrefix = "vote," + this.currentValidAfter;
+          SortedSet<String> fingerprints = new TreeSet<String>();
+          for (Map.Entry<String, String> e :
+              this.missingDescriptors.entrySet()) {
+            if (e.getValue().equals("NA") &&
+                e.getKey().startsWith(voteKeyPrefix)) {
+              String fingerprint = e.getKey().split(",")[2];
+              fingerprints.add(fingerprint);
+            }
           }
-          url = combinedUrl.toString();
-        } else if (url.contains("extra")) {
-          StringBuilder combinedUrl = new StringBuilder(url);
-          int descriptors = 1;
-          while (descriptors++ <= 96 && !urls.isEmpty() &&
-              urls.first().contains("extra")) {
-            url = urls.first();
-            urls.remove(url);
-            requestedUrls.add(url);
-            combinedUrl.append("+" + url.substring("/tor/extra/d/".
-                length()));
+          for (String fingerprint : fingerprints) {
+            this.requestedVotes++;
+            this.downloadedVotes +=
+                this.downloadResourceFromAuthority(authority, 
+                "/tor/status-vote/current/" + fingerprint);
           }
-          url = combinedUrl.toString();
         }
-        String fullUrl = "http://"; + authority + url;
-        byte[] allData = null;
-        if (!downloaded.contains(fullUrl)) {
-          downloaded.add(fullUrl);
-          numDownloaded++;
-          this.dirRequests.put(authority, dirRequests.get(authority) + 1);
-          try {
-            URL u = new URL(fullUrl);
-            HttpURLConnection huc =
-                (HttpURLConnection) u.openConnection();
-            huc.setRequestMethod("GET");
-            huc.connect();
-            int response = huc.getResponseCode();
-            logger.fine("Downloading http://"; + authority + url + " -> "
-                + response);
-            if (response == 200) {
-              BufferedInputStream in = new BufferedInputStream(
-                  huc.getInputStream());
-              ByteArrayOutputStream baos = new ByteArrayOutputStream();
-              int len;
-              byte[] data = new byte[1024];
-              while ((len = in.read(data, 0, 1024)) >= 0) {
-                baos.write(data, 0, len);
-              }
-              in.close();
-              allData = baos.toByteArray();
-            }
-          } catch (IOException e) {
-            remainingDirSources.remove(authority);
-            currentDirSources.remove(authority);
-            if (!remainingDirSources.isEmpty()) {
-              logger.log(Level.FINE, "Failed downloading from "
-                  + authority + "!", e);
+
+        /* Download either all server and extra-info descriptors or only
+         * those that we're missing. Start with server descriptors, then
+         * request extra-info descriptors. */
+        List<String> types = new ArrayList<String>(Arrays.asList(
+            "server,extra".split(",")));
+        for (String type : types) {
+
+          /* Download all server or extra-info descriptors from this
+           * authority if we haven't done so for 24 hours and if we're
+           * configured to do so. */
+          /* TODO Distribute downloads of all descriptors over the day for
+           * different authorities. Maybe limit the number of these
+           * downloads to 1 or 2 per execution. */
+          if (this.downloadAllDescriptorsFromAuthorities.contains(
+              authority) && ((type.equals("server") &&
+              this.downloadAllServerDescriptors) ||
+              (type.equals("extra") && this.downloadAllExtraInfos))) {
+            int downloadedAllDescriptors =
+                this.downloadResourceFromAuthority(authority, "/tor/"
+                + type + "/all");
+            if (type.equals("server")) {
+              this.requestedAllServerDescriptors++;
+              this.downloadedAllServerDescriptors +=
+                  downloadedAllDescriptors;
             } else {
-              logger.log(Level.WARNING, "Failed downloading from "
-                  + authority + "! We have no authorities left to download "
-                  + "from!", e);
+              this.requestedAllExtraInfoDescriptors++;
+              this.downloadedAllExtraInfoDescriptors +=
+                  downloadedAllDescriptors;
             }
-          }
-        }
-        if (allData != null) {
-          if (url.endsWith("consensus")) {
-            this.rdp.parse(allData);
-            this.downloadedConsensuses++;
-          } else if (url.contains("status-vote")) {
-            this.rdp.parse(allData);
-            this.downloadedVotes++;
-          } else if (url.contains("server") ||
-              url.contains("extra")) {
-            String ascii = null;
-            try {
-              ascii = new String(allData, "US-ASCII");
-            } catch (UnsupportedEncodingException e) {
-            }
-            int start = -1, sig = -1, end = -1;
-            String startToken = url.contains("server") ?
-                "router " : "extra-info ";
-            String sigToken = "\nrouter-signature\n";
-            String endToken = "\n-----END SIGNATURE-----\n";
-            while (end < ascii.length()) {
-              start = ascii.indexOf(startToken, end);
-              if (start < 0) {
-                break;
-              }
-              sig = ascii.indexOf(sigToken, start);
-              if (sig < 0) {
-                break;
+
+          /* Download missing server or extra-info descriptors if we're
+           * configured to do so. */
+          } else if ((type.equals("server") &&
+              this.downloadMissingServerDescriptors) ||
+              (type.equals("extra") && this.downloadMissingExtraInfos)) {
+
+            /* Go through the list of missing descriptors of this type
+             * and combine the descriptor identifiers to a URL of up to
+             * 96 descriptors that we can download at once. */
+            SortedSet<String> descriptorIdentifiers =
+                new TreeSet<String>();
+            for (Map.Entry<String, String> e :
+                this.missingDescriptors.entrySet()) {
+              if (e.getValue().equals("NA") &&
+                  e.getKey().startsWith(type + ",") &&
+                  this.descriptorCutOff.compareTo(
+                  e.getKey().split(",")[1]) < 0) {
+                String descriptorIdentifier = e.getKey().split(",")[3];
+                descriptorIdentifiers.add(descriptorIdentifier);
               }
-              sig += sigToken.length();
-              end = ascii.indexOf(endToken, sig);
-              if (end < 0) {
-                break;
+            }
+            StringBuilder combinedResource = null;
+            int descriptorsInCombinedResource = 0,
+                requestedDescriptors = 0, downloadedDescriptors = 0;
+            for (String descriptorIdentifier : descriptorIdentifiers) {
+              if (descriptorsInCombinedResource >= 96) {
+                requestedDescriptors += descriptorsInCombinedResource;
+                downloadedDescriptors +=
+                    this.downloadResourceFromAuthority(authority,
+                    combinedResource.toString());
+                combinedResource = null;
+                descriptorsInCombinedResource = 0;
               }
-              end += endToken.length();
-              byte[] descBytes = new byte[end - start];
-              System.arraycopy(allData, start, descBytes, 0, end - start);
-              String digest = Hex.encodeHexString(DigestUtils.sha(
-                  descBytes));
-              this.rdp.parse(descBytes);
-              if (url.contains("server")) {
-                this.downloadedServerDescriptors++;
-                requestedUrls.remove("/tor/server/d/" + digest);
+              if (descriptorsInCombinedResource == 0) {
+                combinedResource = new StringBuilder("/tor/" + type
+                    + "/d/" + descriptorIdentifier);
               } else {
-                this.downloadedExtraInfoDescriptors++;
-                requestedUrls.remove("/tor/extra/d/" + digest);
+                combinedResource.append("+" + descriptorIdentifier);
               }
+              descriptorsInCombinedResource++;
+            }
+            if (descriptorsInCombinedResource > 0) {
+              requestedDescriptors += descriptorsInCombinedResource;
+              downloadedDescriptors +=
+                  this.downloadResourceFromAuthority(authority,
+                  combinedResource.toString());
+            }
+            if (type.equals("server")) {
+              this.requestedMissingServerDescriptors +=
+                  requestedDescriptors;
+              this.downloadedMissingServerDescriptors +=
+                  downloadedDescriptors;
+            } else {
+              this.requestedMissingExtraInfoDescriptors +=
+                  requestedDescriptors;
+              this.downloadedMissingExtraInfoDescriptors +=
+                  downloadedDescriptors;
             }
           }
         }
-        if (requestedUrls.isEmpty()) {
-          retryUrls.addAll(requestedUrls);
-          logger.fine("Retrying " + requestedUrls.size() + " URLs for "
-              + "which we didn't receive descriptors.");
-        }
-        if (urls.isEmpty()) {
-          currentDirSources.remove(authority);
-          urls.addAll(retryUrls);
-          retryUrls.clear();
-        }
+
+      /* If a download failed, stop requesting descriptors from this
+       * authority and move on to the next. */
+      } catch (IOException e) {
+        logger.log(Level.FINE, "Failed downloading from " + authority
+            + "!", e);
       }
+    }
+  }
 
-      /* If we haven't downloaded a single descriptor in this iteration,
-       * we cannot have learned something new, so we're done. */
-      if (numDownloaded < 1) {
-        break;
+  /**
+   * Attempts to download one or more descriptors identified by a resource
+   * string from a directory authority and passes the returned
+   * descriptor(s) to the <code>RelayDescriptorParser</code> upon success.
+   * Returns the number of descriptors contained in the reply. Throws an
+   * <code>IOException</code> if something goes wrong while downloading.
+   */
+  private int downloadResourceFromAuthority(String authority,
+      String resource) throws IOException {
+    byte[] allData = null;
+    this.requestsByAuthority.put(authority,
+        this.requestsByAuthority.get(authority) + 1);
+    /* TODO Disable compressed downloads for extra-info descriptors,
+     * because zlib decompression doesn't work correctly. Figure out why
+     * this is and fix it. */
+    String fullUrl = "http://"; + authority + resource
+        + (this.downloadCompressed && !resource.startsWith("/tor/extra/")
+        ? ".z" : "");
+    URL u = new URL(fullUrl);
+    HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+    huc.setRequestMethod("GET");
+    huc.connect();
+    int response = huc.getResponseCode();
+    if (response == 200) {
+      BufferedInputStream in = downloadCompressed
+          ? new BufferedInputStream(new InflaterInputStream(
+          huc.getInputStream()))
+          : new BufferedInputStream(huc.getInputStream());
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      int len;
+      byte[] data = new byte[1024];
+      while ((len = in.read(data, 0, 1024)) >= 0) {
+        baos.write(data, 0, len);
+      }
+      in.close();
+      allData = baos.toByteArray();
+    }
+    logger.fine("Downloaded " + fullUrl + " -> " + response + " ("
+        + (allData == null ? 0 : allData.length) + " bytes)");
+    int receivedDescriptors = 0;
+    if (allData != null) {
+      if (resource.startsWith("/tor/status-vote/current/")) {
+        this.rdp.parse(allData);
+        receivedDescriptors = 1;
+      } else if (resource.startsWith("/tor/server/") ||
+          resource.startsWith("/tor/extra/")) {
+        if (resource.equals("/tor/server/all")) {
+          this.lastDownloadedAllDescriptors.put(authority,
+              this.currentTimestamp);
+        }
+        String ascii = null;
+        try {
+          ascii = new String(allData, "US-ASCII");
+        } catch (UnsupportedEncodingException e) {
+          /* No way that US-ASCII is not supported. */
+        }
+        int start = -1, sig = -1, end = -1;
+        String startToken = resource.startsWith("/tor/server/") ?
+            "router " : "extra-info ";
+        String sigToken = "\nrouter-signature\n";
+        String endToken = "\n-----END SIGNATURE-----\n";
+        while (end < ascii.length()) {
+          start = ascii.indexOf(startToken, end);
+          if (start < 0) {
+            break;
+          }
+          sig = ascii.indexOf(sigToken, start);
+          if (sig < 0) {
+            break;
+          }
+          sig += sigToken.length();
+          end = ascii.indexOf(endToken, sig);
+          if (end < 0) {
+            break;
+          }
+          end += endToken.length();
+          byte[] descBytes = new byte[end - start];
+          System.arraycopy(allData, start, descBytes, 0, end - start);
+          String digest = Hex.encodeHexString(DigestUtils.sha(
+              descBytes));
+          this.rdp.parse(descBytes);
+          receivedDescriptors++;
+        }
       }
     }
+    return receivedDescriptors;
   }
 
+  /**
+   * Writes status files to disk and logs statistics about downloading
+   * relay descriptors in this execution.
+   */
   public void writeFile() {
+
+    /* Write missing descriptors file to disk. */
     int missingConsensuses = 0, missingVotes = 0,
         missingServerDescriptors = 0, missingExtraInfoDescriptors = 0;
     try {
@@ -581,8 +704,8 @@ public class RelayDescriptorDownloader {
           this.missingDescriptorsFile));
       for (Map.Entry<String, String> e :
           this.missingDescriptors.entrySet()) {
-        String key = e.getKey();
-        if (!e.getValue().equals("NA")) {
+        String key = e.getKey(), value = e.getValue();
+        if (!value.equals("NA")) {
           /* Not missing. */
         } else if (key.startsWith("consensus,")) {
           missingConsensuses++;
@@ -593,7 +716,7 @@ public class RelayDescriptorDownloader {
         } else if (key.startsWith("extra,")) {
           missingExtraInfoDescriptors++;
         }
-        bw.write(e.getKey() + "," + e.getValue() + "\n");
+        bw.write(key + "," + value + "\n");
       }
       bw.close();
       this.logger.fine("Finished writing file "
@@ -603,35 +726,79 @@ public class RelayDescriptorDownloader {
           + this.missingDescriptorsFile.getAbsolutePath() + "!", e);
     }
 
-    dumpStats.append("During this execution, we added "
+    /* Write text file containing the directory authorities and when we
+     * last downloaded all server and extra-info descriptors from them to
+     * disk. */
+    try {
+      this.logger.fine("Writing file "
+          + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+          + "...");
+      this.lastDownloadedAllDescriptorsFile.getParentFile().mkdirs();
+      BufferedWriter bw = new BufferedWriter(new FileWriter(
+          this.lastDownloadedAllDescriptorsFile));
+      for (Map.Entry<String, String> e :
+          this.lastDownloadedAllDescriptors.entrySet()) {
+        String authority = e.getKey();
+        String lastDownloaded = e.getValue();
+        bw.write(authority + "," + lastDownloaded + "\n");
+      }
+      bw.close();
+      this.logger.fine("Finished writing file "
+          + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+          + ".");
+    } catch (IOException e) {
+      this.logger.log(Level.WARNING, "Failed writing "
+          + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + "!",
+          e);
+    }
+
+    /* Log statistics about this execution. */
+    this.logger.info("Finished downloading relay descriptors from the "
+        + "directory authorities.");
+    this.logger.info("At the beginning of this execution, we were "
+        + "missing " + oldMissingConsensuses + " consensus(es), "
+        + oldMissingVotes + " vote(s), " + oldMissingServerDescriptors
+        + " server descriptor(s), and " + oldMissingExtraInfoDescriptors
+        + " extra-info descriptor(s).");
+    this.logger.info("During this execution, we added "
         + this.newMissingConsensuses + " consensus(es), "
         + this.newMissingVotes + " vote(s), "
         + this.newMissingServerDescriptors + " server descriptor(s), and "
         + this.newMissingExtraInfoDescriptors + " extra-info "
-        + "descriptor(s) to the missing list.\n");
-    dumpStats.append("We attempted to download " + this.triedConsensuses
-        + " consensus(es), " + this.triedVotes + " vote(s), "
-        + this.triedServerDescriptors + " server descriptor(s), and "
-        + this.triedExtraInfoDescriptors + " extra-info descriptor(s) "
-        + "from the directory authorities.\n");
-    dumpStats.append("Requests were sent to these directory "
-        + "authorities:");
-    for (String dirSource : this.dirSources) {
-      dumpStats.append(" " + dirSource + "="
-         + this.dirRequests.get(dirSource));
+        + "descriptor(s) to the missing list, some of which we also "
+        + "requested and removed from the list again.");
+    this.logger.info("We requested " + this.requestedConsensuses
+        + " consensus(es), " + this.requestedVotes + " vote(s), "
+        + this.requestedMissingServerDescriptors + " missing server "
+        + "descriptor(s), " + this.requestedAllServerDescriptors
+        + " times all server descriptors, "
+        + this.requestedMissingExtraInfoDescriptors + " missing "
+        + "extra-info descriptor(s), and "
+        + this.requestedAllExtraInfoDescriptors + " times all extra-info "
+        + "descriptors from the directory authorities.");
+    StringBuilder sb = new StringBuilder();
+    for (String authority : this.authorities) {
+      sb.append(" " + authority + "="
+         + this.requestsByAuthority.get(authority));
     }
-    dumpStats.append("\nWe successfully downloaded "
+    this.logger.info("We sent these numbers of requests to the directory "
+        + "authorities:" + sb.toString());
+    this.logger.info("We successfully downloaded "
         + this.downloadedConsensuses + " consensus(es), "
         + this.downloadedVotes + " vote(s), "
-        + this.downloadedServerDescriptors + " server descriptor(s), and "
-        + this.downloadedExtraInfoDescriptors + " extra-info "
-        + "descriptor(s) from the directory authorities.\n");
-    dumpStats.append("At the end of this execution, "
-      + "we are missing " + missingConsensuses + " consensus(es), "
-      + missingVotes + " vote(s), " + missingServerDescriptors
-      + " server descriptor(s), and " + missingExtraInfoDescriptors
+        + this.downloadedMissingServerDescriptors + " missing server "
+        + "descriptor(s), " + this.downloadedAllServerDescriptors
+        + " server descriptor(s) when downloading all descriptors, "
+        + this.downloadedMissingExtraInfoDescriptors + " missing "
+        + "extra-info descriptor(s) and "
+        + this.downloadedAllExtraInfoDescriptors + " extra-info "
+        + "descriptor(s) when downloading all descriptors.");
+    this.logger.info("At the end of this execution, we are missing "
+      + missingConsensuses + " consensus(es), " + missingVotes
+      + " vote(s), " + missingServerDescriptors + " server "
+      + "descriptor(s), and " + missingExtraInfoDescriptors
       + " extra-info descriptor(s), some of which we may try in the next "
       + "execution.");
-    this.logger.info(dumpStats.toString());
   }
 }
+

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits