[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-db/master] Make check for missing descriptors much more efficient.



commit bee74460bbe58abd8e68a9e88c1ff8d972b4b44a
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Mon Oct 29 19:46:34 2012 -0400

    Make check for missing descriptors much more efficient.
---
 .../ernie/db/relaydescs/ArchiveWriter.java         |  524 ++++++++++++--------
 .../ernie/db/relaydescs/RelayDescriptorParser.java |   18 +-
 2 files changed, 324 insertions(+), 218 deletions(-)

diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
index 8562bd5..a128a16 100644
--- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
+++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
@@ -4,24 +4,29 @@ package org.torproject.ernie.db.relaydescs;
 
 import java.io.BufferedOutputStream;
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.FileReader;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.Stack;
 import java.util.TimeZone;
-import java.util.TreeSet;
+import java.util.TreeMap;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Hex;
 import org.torproject.descriptor.DescriptorParser;
 import org.torproject.descriptor.DescriptorSourceFactory;
 import org.torproject.descriptor.impl.DescriptorParseException;
@@ -35,11 +40,141 @@ public class ArchiveWriter extends Thread {
     this.config = config;
   }
 
+  private long now = System.currentTimeMillis();
   private Logger logger;
   private File outputDirectory;
   private DescriptorParser descriptorParser;
-  private int storedConsensuses = 0, storedVotes = 0, storedCerts = 0,
-      storedServerDescriptors = 0, storedExtraInfoDescriptors = 0;
+  private int storedConsensusesCounter = 0, storedVotesCounter = 0,
+      storedCertsCounter = 0, storedServerDescriptorsCounter = 0,
+      storedExtraInfoDescriptorsCounter = 0;
+
+  private SortedMap<Long, SortedSet<String>> storedConsensuses =
+      new TreeMap<Long, SortedSet<String>>();
+  private SortedMap<Long, Integer> expectedVotes =
+      new TreeMap<Long, Integer>();
+  private SortedMap<Long, SortedMap<String, SortedSet<String>>>
+      storedVotes =
+      new TreeMap<Long, SortedMap<String, SortedSet<String>>>();
+  private SortedMap<Long, Map<String, String>> storedServerDescriptors =
+      new TreeMap<Long, Map<String, String>>();
+  private SortedMap<Long, Set<String>> storedExtraInfoDescriptors =
+      new TreeMap<Long, Set<String>>();
+
+  private File storedServerDescriptorsFile = new File(
+      "stats/stored-server-descriptors");
+  private File storedExtraInfoDescriptorsFile = new File(
+      "stats/stored-extra-info-descriptors");
+
+  private void loadDescriptorDigests() {
+    SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    try {
+      if (this.storedServerDescriptorsFile.exists()) {
+        BufferedReader br = new BufferedReader(new FileReader(
+            this.storedServerDescriptorsFile));
+        String line;
+        while ((line = br.readLine()) != null) {
+          String[] parts = line.split(",");
+          if (parts.length != 3) {
+            this.logger.warning("Could not load server descriptor "
+                + "digests because of illegal line '" + line + "'.  We "
+                + "might not be able to correctly check descriptors for "
+                + "completeness.");
+            break;
+          }
+          long published = dateTimeFormat.parse(parts[0]).getTime();
+          if (published < this.now - 48L * 60L * 60L * 1000L) {
+            continue;
+          }
+          if (!this.storedServerDescriptors.containsKey(published)) {
+            this.storedServerDescriptors.put(published,
+                new HashMap<String, String>());
+          }
+          String serverDescriptorDigest = parts[1];
+          String extraInfoDescriptorDigest = parts[2].equals("NA") ? null
+              : parts[2];
+          this.storedServerDescriptors.get(published).put(
+              serverDescriptorDigest, extraInfoDescriptorDigest);
+        }
+        br.close();
+      }
+      if (this.storedExtraInfoDescriptorsFile.exists()) {
+        BufferedReader br = new BufferedReader(new FileReader(
+            this.storedExtraInfoDescriptorsFile));
+        String line;
+        while ((line = br.readLine()) != null) {
+          String[] parts = line.split(",");
+          if (parts.length != 2) {
+            this.logger.warning("Could not load extra-info descriptor "
+                + "digests because of illegal line '" + line + "'.  We "
+                + "might not be able to correctly check descriptors for "
+                + "completeness.");
+            break;
+          }
+          long published = dateTimeFormat.parse(parts[0]).getTime();
+          if (published < this.now - 48L * 60L * 60L * 1000L) {
+            continue;
+          }
+          if (!this.storedExtraInfoDescriptors.containsKey(published)) {
+            this.storedExtraInfoDescriptors.put(published,
+                new HashSet<String>());
+          }
+          String extraInfoDescriptorDigest = parts[1];
+          this.storedExtraInfoDescriptors.get(published).add(
+              extraInfoDescriptorDigest);
+        }
+        br.close();
+      }
+    } catch (ParseException e) {
+      this.logger.log(Level.WARNING, "Could not load descriptor "
+          + "digests.  We might not be able to correctly check "
+          + "descriptors for completeness.", e);
+    } catch (IOException e) {
+      this.logger.log(Level.WARNING, "Could not load descriptor "
+          + "digests.  We might not be able to correctly check "
+          + "descriptors for completeness.", e);
+    }
+  }
+
+  private void saveDescriptorDigests() {
+    SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    try {
+      this.storedServerDescriptorsFile.getParentFile().mkdirs();
+      BufferedWriter bw = new BufferedWriter(new FileWriter(
+          this.storedServerDescriptorsFile));
+      for (Map.Entry<Long, Map<String, String>> e :
+          this.storedServerDescriptors.entrySet()) {
+        String published = dateTimeFormat.format(e.getKey());
+        for (Map.Entry<String, String> f : e.getValue().entrySet()) {
+          String serverDescriptorDigest = f.getKey();
+          String extraInfoDescriptorDigest = f.getValue() == null ? "NA"
+              : f.getValue();
+          bw.write(String.format("%s,%s,%s%n", published,
+              serverDescriptorDigest, extraInfoDescriptorDigest));
+        }
+      }
+      bw.close();
+      this.storedExtraInfoDescriptorsFile.getParentFile().mkdirs();
+      bw = new BufferedWriter(new FileWriter(
+          this.storedExtraInfoDescriptorsFile));
+      for (Map.Entry<Long, Set<String>> e :
+          this.storedExtraInfoDescriptors.entrySet()) {
+        String published = dateTimeFormat.format(e.getKey());
+        for (String extraInfoDescriptorDigest : e.getValue()) {
+          bw.write(String.format("%s,%s%n", published,
+              extraInfoDescriptorDigest));
+        }
+      }
+      bw.close();
+    } catch (IOException e) {
+      this.logger.log(Level.WARNING, "Could not save descriptor "
+          + "digests.  We might not be able to correctly check "
+          + "descriptors for completeness in the next run.", e);
+    }
+  }
 
   public void run() {
 
@@ -52,6 +187,8 @@ public class ArchiveWriter extends Thread {
     this.descriptorParser =
         DescriptorSourceFactory.createDescriptorParser();
 
+    this.loadDescriptorDigests();
+
     // Prepare relay descriptor parser
     RelayDescriptorParser rdp = new RelayDescriptorParser(this);
 
@@ -92,12 +229,13 @@ public class ArchiveWriter extends Thread {
           + "directory authorities");
     }
 
-    // Write output to disk that only depends on relay descriptors
-    this.dumpStats();
+    this.checkMissingDescriptors();
 
     this.checkStaledescriptors();
 
     this.cleanUpRsyncDirectory();
+
+    this.saveDescriptorDigests();
   }
 
   private boolean store(byte[] typeAnnotation, byte[] data,
@@ -131,12 +269,11 @@ public class ArchiveWriter extends Thread {
     return false;
   }
 
-  private long maxConsensusValidAfter = 0L;
   private static final byte[] CONSENSUS_ANNOTATION =
       "@type network-status-consensus-3 1.0\n".getBytes();
-  public void storeConsensus(byte[] data, long validAfter) {
-    this.maxConsensusValidAfter = Math.max(this.maxConsensusValidAfter,
-        validAfter);
+  public void storeConsensus(byte[] data, long validAfter,
+      SortedSet<String> dirSources,
+      SortedSet<String> serverDescriptorDigests) {
     SimpleDateFormat printFormat = new SimpleDateFormat(
         "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
     printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -146,16 +283,22 @@ public class ArchiveWriter extends Thread {
         + tarballFile.getName());
     File[] outputFiles = new File[] { tarballFile, rsyncFile };
     if (this.store(CONSENSUS_ANNOTATION, data, outputFiles)) {
-      this.storedConsensuses++;
+      this.storedConsensusesCounter++;
+    }
+    SimpleDateFormat dateTimeFormat =
+        new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    if (this.now - validAfter < 3L * 60L * 60L * 1000L) {
+      this.storedConsensuses.put(validAfter, serverDescriptorDigests);
+      this.expectedVotes.put(validAfter, dirSources.size());
     }
   }
 
-  private long maxVoteValidAfter = 0L;
   private static final byte[] VOTE_ANNOTATION =
       "@type network-status-vote-3 1.0\n".getBytes();
   public void storeVote(byte[] data, long validAfter,
-      String fingerprint, String digest) {
-    this.maxVoteValidAfter = Math.max(this.maxVoteValidAfter, validAfter);
+      String fingerprint, String digest,
+      SortedSet<String> serverDescriptorDigests) {
     SimpleDateFormat printFormat = new SimpleDateFormat(
         "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
     printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -166,7 +309,18 @@ public class ArchiveWriter extends Thread {
         + tarballFile.getName());
     File[] outputFiles = new File[] { tarballFile, rsyncFile };
     if (this.store(VOTE_ANNOTATION, data, outputFiles)) {
-      this.storedVotes++;
+      this.storedVotesCounter++;
+    }
+    SimpleDateFormat dateTimeFormat =
+        new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    if (this.now - validAfter < 3L * 60L * 60L * 1000L) {
+      if (!this.storedVotes.containsKey(validAfter)) {
+        this.storedVotes.put(validAfter,
+            new TreeMap<String, SortedSet<String>>());
+      }
+      this.storedVotes.get(validAfter).put(fingerprint,
+          serverDescriptorDigests);
     }
   }
 
@@ -181,17 +335,14 @@ public class ArchiveWriter extends Thread {
         + fingerprint + "-" + printFormat.format(new Date(published)));
     File[] outputFiles = new File[] { tarballFile };
     if (this.store(CERTIFICATE_ANNOTATION, data, outputFiles)) {
-      this.storedCerts++;
+      this.storedCertsCounter++;
     }
   }
 
-  private long maxServerDescriptorPublished = 0L;
   private static final byte[] SERVER_DESCRIPTOR_ANNOTATION =
       "@type server-descriptor 1.0\n".getBytes();
   public void storeServerDescriptor(byte[] data, String digest,
-      long published) {
-    this.maxServerDescriptorPublished = Math.max(
-        this.maxServerDescriptorPublished, published);
+      long published, String extraInfoDigest) {
     SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
     printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
     File tarballFile = new File(this.outputDirectory
@@ -202,17 +353,25 @@ public class ArchiveWriter extends Thread {
         "rsync/relay-descriptors/server-descriptors/" + digest);
     File[] outputFiles = new File[] { tarballFile, rsyncFile };
     if (this.store(SERVER_DESCRIPTOR_ANNOTATION, data, outputFiles)) {
-      this.storedServerDescriptors++;
+      this.storedServerDescriptorsCounter++;
+    }
+    SimpleDateFormat dateTimeFormat =
+        new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    if (this.now - published < 48L * 60L * 60L * 1000L) {
+      if (!this.storedServerDescriptors.containsKey(published)) {
+        this.storedServerDescriptors.put(published,
+            new HashMap<String, String>());
+      }
+      this.storedServerDescriptors.get(published).put(digest,
+          extraInfoDigest);
     }
   }
 
-  private long maxExtraInfoDescriptorPublished = 0L;
   private static final byte[] EXTRA_INFO_ANNOTATION =
       "@type extra-info 1.0\n".getBytes();
   public void storeExtraInfoDescriptor(byte[] data,
       String extraInfoDigest, long published) {
-    this.maxExtraInfoDescriptorPublished = Math.max(
-        this.maxExtraInfoDescriptorPublished, published);
     SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
     descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
     File tarballFile = new File(this.outputDirectory + "/extra-info/"
@@ -224,196 +383,143 @@ public class ArchiveWriter extends Thread {
         + extraInfoDigest);
     File[] outputFiles = new File[] { tarballFile, rsyncFile };
     if (this.store(EXTRA_INFO_ANNOTATION, data, outputFiles)) {
-      this.storedExtraInfoDescriptors++;
+      this.storedExtraInfoDescriptorsCounter++;
+    }
+    if (this.now - published < 48L * 60L * 60L * 1000L) {
+      if (!this.storedExtraInfoDescriptors.containsKey(published)) {
+        this.storedExtraInfoDescriptors.put(published,
+            new HashSet<String>());
+      }
+      this.storedExtraInfoDescriptors.get(published).add(extraInfoDigest);
     }
   }
 
   private StringBuilder intermediateStats = new StringBuilder();
   public void intermediateStats(String event) {
     intermediateStats.append("While " + event + ", we stored "
-        + this.storedConsensuses + " consensus(es), " + this.storedVotes
-        + " vote(s), " + this.storedCerts + " certificate(s), "
-        + this.storedServerDescriptors + " server descriptor(s), and "
-        + this.storedExtraInfoDescriptors
+        + this.storedConsensusesCounter + " consensus(es), "
+        + this.storedVotesCounter + " vote(s), " + this.storedCertsCounter
+        + " certificate(s), " + this.storedServerDescriptorsCounter
+        + " server descriptor(s), and "
+        + this.storedExtraInfoDescriptorsCounter
         + " extra-info descriptor(s) to disk.\n");
-    this.storedConsensuses = 0;
-    this.storedVotes = 0;
-    this.storedCerts = 0;
-    this.storedServerDescriptors = 0;
-    this.storedExtraInfoDescriptors = 0;
+    this.storedConsensusesCounter = 0;
+    this.storedVotesCounter = 0;
+    this.storedCertsCounter = 0;
+    this.storedServerDescriptorsCounter = 0;
+    this.storedExtraInfoDescriptorsCounter = 0;
   }
-  /**
-   * Dump some statistics on the completeness of descriptors to the logs
-   * on level INFO.
-   */
-  public void dumpStats() {
+
+  private void checkMissingDescriptors() {
     StringBuilder sb = new StringBuilder("Finished writing relay "
         + "descriptors to disk.\n");
     sb.append(intermediateStats.toString());
     sb.append("Statistics on the completeness of written relay "
         + "descriptors of the last 3 consensuses (Consensus/Vote, "
         + "valid-after, votes, server descriptors, extra-infos):");
-    try {
-      SimpleDateFormat validAfterFormat =
-          new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-      validAfterFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-      SimpleDateFormat consensusVoteFormat =
-          new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
-      consensusVoteFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-      SimpleDateFormat descriptorFormat =
-          new SimpleDateFormat("yyyy/MM/");
-      descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-
-      SortedSet<File> consensuses = new TreeSet<File>();
-      Stack<File> leftToParse = new Stack<File>();
-      leftToParse.add(new File(outputDirectory + "/consensus"));
-      while (!leftToParse.isEmpty()) {
-        File pop = leftToParse.pop();
-        if (pop.isDirectory()) {
-          for (File f : pop.listFiles()) {
-            leftToParse.add(f);
-          }
-        } else if (pop.length() > 0) {
-          consensuses.add(pop);
-        }
-        while (consensuses.size() > 3) {
-          consensuses.remove(consensuses.first());
-        }
-      }
-      for (File f : consensuses) {
-        BufferedReader br = new BufferedReader(new FileReader(f));
-        String line = null, validAfterTime = null,
-            voteFilenamePrefix = null, dirSource = null;
-        int allVotes = 0, foundVotes = 0,
-            allServerDescs = 0, foundServerDescs = 0,
-            allExtraInfos = 0, foundExtraInfos = 0;
-        while ((line = br.readLine()) != null) {
-          if (line.startsWith("valid-after ")) {
-            validAfterTime = line.substring("valid-after ".length());
-            long validAfter = validAfterFormat.parse(
-                validAfterTime).getTime();
-            voteFilenamePrefix = outputDirectory + "/vote/"
-                + consensusVoteFormat.format(new Date(validAfter))
-                + "-vote-";
-          } else if (line.startsWith("dir-source ")) {
-            dirSource = line.split(" ")[2];
-          } else if (line.startsWith("vote-digest ")) {
-            allVotes++;
-            File voteFile = new File(voteFilenamePrefix + dirSource + "-"
-                + line.split(" ")[1]);
-            if (voteFile.exists()) {
-              foundVotes++;
-              BufferedReader vbr = new BufferedReader(new FileReader(
-                  voteFile));
-              String line3 = null;
-              int voteAllServerDescs = 0, voteFoundServerDescs = 0,
-                  voteAllExtraInfos = 0, voteFoundExtraInfos = 0;
-              while ((line3 = vbr.readLine()) != null) {
-                if (line3.startsWith("r ")) {
-                  voteAllServerDescs++;
-                  String digest = Hex.encodeHexString(Base64.decodeBase64(
-                      line3.split(" ")[3] + "=")).toLowerCase();
-                  long published = validAfterFormat.parse(
-                      line3.split(" ")[4] + " "
-                      + line3.split(" ")[5]).getTime();
-                  String filename = outputDirectory
-                      + "/server-descriptor/"
-                      + descriptorFormat.format(new Date(published))
-                      + digest.substring(0, 1) + "/"
-                      + digest.substring(1, 2) + "/" + digest;
-                  if (new File(filename).exists()) {
-                    BufferedReader sbr = new BufferedReader(new FileReader(
-                        new File(filename)));
-                    String line2 = null;
-                    while ((line2 = sbr.readLine()) != null) {
-                      if (line2.startsWith("opt extra-info-digest ") ||
-                          line2.startsWith("extra-info-digest ")) {
-                        voteAllExtraInfos++;
-                        String extraInfoDigest = line2.startsWith("opt ") ?
-                            line2.split(" ")[2].toLowerCase() :
-                            line2.split(" ")[1].toLowerCase();
-                        String filename2 =
-                            outputDirectory.getAbsolutePath()
-                            + "/extra-info/"
-                            + descriptorFormat.format(new Date(published))
-                            + extraInfoDigest.substring(0, 1) + "/"
-                            + extraInfoDigest.substring(1, 2) + "/"
-                            + extraInfoDigest;
-                        if (new File(filename2).exists()) {
-                          voteFoundExtraInfos++;
-                        }
-                      }
-                    }
-                    sbr.close();
-                    voteFoundServerDescs++;
-                  }
+    SimpleDateFormat dateTimeFormat =
+        new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    Map<String, String> knownServerDescriptors =
+        new HashMap<String, String>();
+    for (Map<String, String> descriptors :
+        this.storedServerDescriptors.values()) {
+      knownServerDescriptors.putAll(descriptors);
+    }
+    Set<String> knownExtraInfoDescriptors = new HashSet<String>();
+    for (Set<String> descriptors :
+        this.storedExtraInfoDescriptors.values()) {
+      knownExtraInfoDescriptors.addAll(descriptors);
+    }
+    boolean missingDescriptors = false, missingVotes = false;
+    for (Map.Entry<Long, SortedSet<String>> c :
+        this.storedConsensuses.entrySet()) {
+      long validAfterMillis = c.getKey();
+      String validAfterTime = dateTimeFormat.format(validAfterMillis);
+      int allVotes = this.expectedVotes.containsKey(validAfterMillis)
+          ? this.expectedVotes.get(validAfterMillis) : 0;
+      int foundVotes = 0;
+      if (this.storedVotes.containsKey(validAfterMillis)) {
+        foundVotes = this.storedVotes.get(validAfterMillis).size();
+        for (Map.Entry<String, SortedSet<String>> v :
+            this.storedVotes.get(validAfterMillis).entrySet()) {
+          int voteFoundServerDescs = 0, voteAllServerDescs = 0,
+              voteFoundExtraInfos = 0, voteAllExtraInfos = 0;
+          for (String serverDescriptorDigest : v.getValue()) {
+            voteAllServerDescs++;
+            if (knownServerDescriptors.containsKey(
+                serverDescriptorDigest)) {
+              voteFoundServerDescs++;
+              if (knownServerDescriptors.get(serverDescriptorDigest)
+                  != null) {
+                String extraInfoDescriptorDigest =
+                    knownServerDescriptors.get(serverDescriptorDigest);
+                voteAllExtraInfos++;
+                if (knownExtraInfoDescriptors.contains(
+                    extraInfoDescriptorDigest)) {
+                  voteFoundExtraInfos++;
                 }
               }
-              vbr.close();
-              sb.append(String.format("%nV, %s, NA, %d/%d (%.1f%%), "
-                  + "%d/%d (%.1f%%)", validAfterTime,
-                  voteFoundServerDescs, voteAllServerDescs,
-                  100.0D * (double) voteFoundServerDescs /
-                    (double) voteAllServerDescs,
-                  voteFoundExtraInfos, voteAllExtraInfos,
-                  100.0D * (double) voteFoundExtraInfos /
-                    (double) voteAllExtraInfos));
             }
-          } else if (line.startsWith("r ")) {
-            allServerDescs++;
-            String digest = Hex.encodeHexString(Base64.decodeBase64(
-                line.split(" ")[3] + "=")).toLowerCase();
-            long published = validAfterFormat.parse(
-                line.split(" ")[4] + " " + line.split(" ")[5]).getTime();
-            String filename = outputDirectory.getAbsolutePath()
-                + "/server-descriptor/"
-                + descriptorFormat.format(new Date(published))
-                + digest.substring(0, 1) + "/"
-                + digest.substring(1, 2) + "/" + digest;
-            if (new File (filename).exists()) {
-              BufferedReader sbr = new BufferedReader(new FileReader(
-                  new File(filename)));
-              String line2 = null;
-              while ((line2 = sbr.readLine()) != null) {
-                if (line2.startsWith("opt extra-info-digest ") ||
-                    line2.startsWith("extra-info-digest ")) {
-                  allExtraInfos++;
-                  String extraInfoDigest = line2.startsWith("opt ") ?
-                      line2.split(" ")[2].toLowerCase() :
-                      line2.split(" ")[1].toLowerCase();
-                  String filename2 = outputDirectory.getAbsolutePath()
-                      + "/extra-info/"
-                      + descriptorFormat.format(new Date(published))
-                      + extraInfoDigest.substring(0, 1) + "/"
-                      + extraInfoDigest.substring(1, 2) + "/"
-                      + extraInfoDigest;
-                  if (new File (filename2).exists()) {
-                    foundExtraInfos++;
-                  }
-                }
-              }
-              sbr.close();
-              foundServerDescs++;
+          }
+          sb.append(String.format("%nV, %s, NA, %d/%d (%.1f%%), "
+              + "%d/%d (%.1f%%)", validAfterTime,
+              voteFoundServerDescs, voteAllServerDescs,
+              100.0D * (double) voteFoundServerDescs /
+                (double) voteAllServerDescs,
+              voteFoundExtraInfos, voteAllExtraInfos,
+              100.0D * (double) voteFoundExtraInfos /
+                (double) voteAllExtraInfos));
+          if (voteFoundServerDescs * 1000 < voteAllServerDescs * 999 ||
+              voteFoundExtraInfos * 1000 < voteAllExtraInfos * 999) {
+            missingDescriptors = true;
+          }
+        }
+      }
+      int foundServerDescs = 0, allServerDescs = 0, foundExtraInfos = 0,
+          allExtraInfos = 0;
+      for (String serverDescriptorDigest : c.getValue()) {
+        allServerDescs++;
+        if (knownServerDescriptors.containsKey(
+            serverDescriptorDigest)) {
+          foundServerDescs++;
+          if (knownServerDescriptors.get(
+              serverDescriptorDigest) != null) {
+            allExtraInfos++;
+            String extraInfoDescriptorDigest =
+                knownServerDescriptors.get(serverDescriptorDigest);
+            if (knownExtraInfoDescriptors.contains(
+                extraInfoDescriptorDigest)) {
+              foundExtraInfos++;
             }
           }
         }
-        br.close();
-        sb.append(String.format("%nC, %s, %d/%d (%.1f%%), "
-            + "%d/%d (%.1f%%), %d/%d (%.1f%%)",
-            validAfterTime, foundVotes, allVotes,
-            100.0D * (double) foundVotes / (double) allVotes,
-            foundServerDescs, allServerDescs,
-            100.0D * (double) foundServerDescs / (double) allServerDescs,
-            foundExtraInfos, allExtraInfos,
-            100.0D * (double) foundExtraInfos / (double) allExtraInfos));
       }
-      this.logger.info(sb.toString());
-    } catch (IOException e) {
-      this.logger.log(Level.WARNING, "Could not dump statistics to disk.",
-          e);
-    } catch (ParseException e) {
-      this.logger.log(Level.WARNING, "Could not dump statistics to disk.",
-          e);
+      sb.append(String.format("%nC, %s, %d/%d (%.1f%%), "
+          + "%d/%d (%.1f%%), %d/%d (%.1f%%)",
+          validAfterTime, foundVotes, allVotes,
+          100.0D * (double) foundVotes / (double) allVotes,
+          foundServerDescs, allServerDescs,
+          100.0D * (double) foundServerDescs / (double) allServerDescs,
+          foundExtraInfos, allExtraInfos,
+          100.0D * (double) foundExtraInfos / (double) allExtraInfos));
+      if (foundServerDescs * 1000 < allServerDescs * 999 ||
+          foundExtraInfos * 1000 < allExtraInfos * 999) {
+        missingDescriptors = true;
+      }
+      if (foundVotes < allVotes) {
+        missingVotes = true;
+      }
+    }
+    this.logger.info(sb.toString());
+    if (missingDescriptors) {
+      this.logger.warning("We are missing at least 0.1% of server or "
+          + "extra-info descriptors referenced from a consensus or "
+          + "vote.");
+    }
+    if (missingVotes) {
+      this.logger.warning("We are missing at least one vote that was "
+          + "referenced from a consensus.");
     }
   }
 
@@ -421,31 +527,33 @@ public class ArchiveWriter extends Thread {
     SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
         "yyyy-MM-dd HH:mm:ss");
     dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    long tooOldMillis = System.currentTimeMillis() - 330L * 60L * 1000L;
-    if (maxConsensusValidAfter > 0L &&
-        maxConsensusValidAfter < tooOldMillis) {
+    long tooOldMillis = this.now - 330L * 60L * 1000L;
+    if (!this.storedConsensuses.isEmpty() &&
+        this.storedConsensuses.lastKey() < tooOldMillis) {
       this.logger.warning("The last known relay network status "
           + "consensus was valid after "
-          + dateTimeFormat.format(maxConsensusValidAfter)
+          + dateTimeFormat.format(this.storedConsensuses.lastKey())
           + ", which is more than 5:30 hours in the past.");
     }
-    if (maxVoteValidAfter > 0L && maxVoteValidAfter < tooOldMillis) {
+    if (!this.storedVotes.isEmpty() &&
+        this.storedVotes.lastKey() < tooOldMillis) {
       this.logger.warning("The last known relay network status vote "
-          + "was valid after " + dateTimeFormat.format(maxVoteValidAfter)
-          + ", which is more than 5:30 hours in the past.");
+          + "was valid after " + dateTimeFormat.format(
+          this.storedVotes.lastKey()) + ", which is more than 5:30 hours "
+          + "in the past.");
     }
-    if (maxServerDescriptorPublished > 0L &&
-        maxServerDescriptorPublished < tooOldMillis) {
+    if (!this.storedServerDescriptors.isEmpty() &&
+        this.storedServerDescriptors.lastKey() < tooOldMillis) {
       this.logger.warning("The last known relay server descriptor was "
           + "published at "
-          + dateTimeFormat.format(maxServerDescriptorPublished)
+          + dateTimeFormat.format(this.storedServerDescriptors.lastKey())
           + ", which is more than 5:30 hours in the past.");
     }
-    if (maxExtraInfoDescriptorPublished > 0L &&
-        maxExtraInfoDescriptorPublished < tooOldMillis) {
+    if (!this.storedExtraInfoDescriptors.isEmpty() &&
+        this.storedExtraInfoDescriptors.lastKey() < tooOldMillis) {
       this.logger.warning("The last known relay extra-info descriptor "
-          + "was published at "
-          + dateTimeFormat.format(maxExtraInfoDescriptorPublished)
+          + "was published at " + dateTimeFormat.format(
+          this.storedExtraInfoDescriptors.lastKey())
           + ", which is more than 5:30 hours in the past.");
     }
   }
diff --git a/src/org/torproject/ernie/db/relaydescs/RelayDescriptorParser.java b/src/org/torproject/ernie/db/relaydescs/RelayDescriptorParser.java
index 6f04c20..107ba73 100644
--- a/src/org/torproject/ernie/db/relaydescs/RelayDescriptorParser.java
+++ b/src/org/torproject/ernie/db/relaydescs/RelayDescriptorParser.java
@@ -82,16 +82,13 @@ public class RelayDescriptorParser {
           new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
       parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
       if (line.equals("network-status-version 3")) {
-        // TODO when parsing the current consensus, check the fresh-until
-        // time to see when we switch from hourly to half-hourly
-        // consensuses
         boolean isConsensus = true;
         String validAfterTime = null, fingerprint = null,
             dirSource = null;
         long validAfter = -1L, dirKeyPublished = -1L;
         SortedSet<String> dirSources = new TreeSet<String>();
         SortedSet<String> serverDescriptors = new TreeSet<String>();
-        SortedSet<String> hashedRelayIdentities = new TreeSet<String>();
+        SortedSet<String> serverDescriptorDigests = new TreeSet<String>();
         StringBuilder certificateStringBuilder = null;
         String certificateString = null;
         while ((line = br.readLine()) != null) {
@@ -137,9 +134,7 @@ public class RelayDescriptorParser {
                 parts[3] + "=")).toLowerCase();
             serverDescriptors.add(publishedTime + "," + relayIdentity
                 + "," + serverDesc);
-            hashedRelayIdentities.add(DigestUtils.shaHex(
-                Base64.decodeBase64(parts[2] + "=")).
-                toUpperCase());
+            serverDescriptorDigests.add(serverDesc);
           }
         }
         if (isConsensus) {
@@ -148,7 +143,8 @@ public class RelayDescriptorParser {
                 serverDescriptors);
           }
           if (this.aw != null) {
-            this.aw.storeConsensus(data, validAfter);
+            this.aw.storeConsensus(data, validAfter, dirSources,
+                serverDescriptorDigests);
           }
         } else {
           if (this.aw != null || this.rdd != null) {
@@ -163,7 +159,8 @@ public class RelayDescriptorParser {
               System.arraycopy(data, start, forDigest, 0, sig - start);
               String digest = DigestUtils.shaHex(forDigest).toUpperCase();
               if (this.aw != null) {
-                this.aw.storeVote(data, validAfter, dirSource, digest);
+                this.aw.storeVote(data, validAfter, dirSource, digest,
+                    serverDescriptorDigests);
               }
               if (this.rdd != null) {
                 this.rdd.haveParsedVote(validAfterTime, fingerprint,
@@ -210,7 +207,8 @@ public class RelayDescriptorParser {
           digest = DigestUtils.shaHex(forDigest);
         }
         if (this.aw != null && digest != null) {
-          this.aw.storeServerDescriptor(data, digest, published);
+          this.aw.storeServerDescriptor(data, digest, published,
+              extraInfoDigest);
         }
         if (this.rdd != null && digest != null) {
           this.rdd.haveParsedServerDescriptor(publishedTime,

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits