[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] [ernie/master] Move missing-descriptors logic to descriptor downloader.
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed, 17 Mar 2010 22:50:51 +0100
Subject: Move missing-descriptors logic to descriptor downloader.
Commit: bdd30a8b44702d5111a45b47bc720902886705f7
---
config | 3 -
src/ArchiveWriter.java | 526 ++++------------------------------
src/CachedRelayDescriptorReader.java | 19 +--
src/Configuration.java | 16 -
src/Main.java | 62 +++--
src/RelayDescriptorDownloader.java | 482 ++++++++++++++++++++++++++-----
src/RelayDescriptorParser.java | 478 +++++++++++++++++--------------
7 files changed, 767 insertions(+), 819 deletions(-)
diff --git a/config b/config
index 5c202f1..18c5ed6 100644
--- a/config
+++ b/config
@@ -49,9 +49,6 @@
## Write directory archives to disk
#WriteDirectoryArchives 0
-## V3 directory authority fingerprints
-#V3DirectoryAuthorities 14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,D586D18309DED4CD6D57C18FDB97EFA96D330566,585769C78764D58426B8B52B6651A5A71137189A,27B6B5996C426270A5C95488AA5BCEB6BCC86956,80550987E1D626E3EBA5E5E75A458DE0626D088C,ED03BB616EB2F60BEC80151114BB25CEF515B226,81349FC1F2DBA2C2C11B45CB9706637D480AB913,E2A2AF570166665D738736D0DD58169CC61D8A8B
-
## Read cached-* files from a local Tor client
#ImportCachedRelayDescriptors 1
diff --git a/src/ArchiveWriter.java b/src/ArchiveWriter.java
index 63a8477..628025c 100644
--- a/src/ArchiveWriter.java
+++ b/src/ArchiveWriter.java
@@ -1,488 +1,72 @@
import java.io.*;
-import java.text.*;
-import java.util.*;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
import java.util.logging.*;
-import org.apache.commons.codec.digest.*;
-import org.apache.commons.codec.binary.*;
public class ArchiveWriter {
- private SortedSet<String> v3DirectoryAuthorities;
- private File missingDescriptorsFile;
- private SortedSet<String> missingDescriptors;
- private boolean missingDescriptorsFileModified = false;
private Logger logger;
- public ArchiveWriter(SortedSet<String> v3DirectoryAuthorities) {
- this.v3DirectoryAuthorities = v3DirectoryAuthorities;
- this.missingDescriptorsFile = new File(
- "stats/archive-writer-parse-history");
+ public ArchiveWriter() {
this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
- SimpleDateFormat parseFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- this.missingDescriptors = new TreeSet<String>();
- SimpleDateFormat consensusVoteFormat =
- new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
- consensusVoteFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- SimpleDateFormat descriptorFormat =
- new SimpleDateFormat("yyyy/MM/");
- descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (this.missingDescriptorsFile.exists()) {
- this.logger.fine("Reading file "
- + this.missingDescriptorsFile.getAbsolutePath() + "...");
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- this.missingDescriptorsFile));
- String line = null;
- long now = System.currentTimeMillis();
- while ((line = br.readLine()) != null) {
- // only add to download list if descriptors are still available
- // on directories
- long published = parseFormat.parse(line.split(",")[2]).
- getTime();
- if (line.startsWith("consensus") &&
- published + 55L * 60L * 1000L > now) {
- File consensusFile = new File("directory-archive/consensus/"
- + consensusVoteFormat.format(new Date(published))
- + "-consensus");
- if (!consensusFile.exists()) {
- this.logger.finer("Initializing missing list with "
- + "consensus: valid-after=" + line.split(",")[2]
- + ", filename=directory-archive/consensus/"
- + consensusVoteFormat.format(new Date(published))
- + "-consensus");
- this.missingDescriptors.add(line);
- }
- } else if (line.startsWith("vote") &&
- published + 55L * 60L * 1000L > now) {
- // TODO is vote even available for 55 minutes after its
- // publication?
- File voteFile = new File("directory-archive/vote/"
- + consensusVoteFormat.format(new Date(published))
- + "-vote-" + line.split(",")[1]);
- File voteFileDir = voteFile.getParentFile();
- String voteFileName = voteFile.getName();
- boolean voteFileFound = false;
- if (voteFileDir.exists()) {
- for (File f : Arrays.asList(voteFileDir.listFiles())) {
- if (f.getName().startsWith(voteFileName)) {
- voteFileFound = true;
- break;
- }
- }
- }
- if (!voteFileFound) {
- this.logger.finer("Initializing missing list with vote: "
- + "fingerprint=" + line.split(",")[1]
- + ", valid-after="
- + consensusVoteFormat.format(new Date(published))
- + ", filename=directory-archive/vote/"
- + consensusVoteFormat.format(new Date(published))
- + "-vote-" + line.split(",")[1] + "-*");
- this.missingDescriptors.add(line);
- }
- } else if ((line.startsWith("server") ||
- line.startsWith("extra")) &&
- published + 24L * 60L * 60L * 1000L > now) {
- // TODO are 24 hours okay?
- boolean isServerDesc = line.startsWith("server");
- String digest = line.split(",")[1].toLowerCase();
- File descriptorFile = new File("directory-archive/"
- + (isServerDesc ? "server-descriptor" : "extra-info")
- + "/" + descriptorFormat.format(new Date(published))
- + digest.substring(0, 1) + "/" + digest.substring(1, 2)
- + "/" + digest);
- if (!descriptorFile.exists()) {
- this.logger.finer("Initializing missing list with "
- + (isServerDesc ? "server" : "extra-info")
- + " descriptor: digest=" + digest
- + ", filename=directory-archive/server-descriptor/"
- + descriptorFormat.format(new Date(published))
- + line.split(",")[1].substring(0, 1) + "/"
- + line.split(",")[1].substring(1, 2) + "/"
- + line.split(",")[1]);
- this.missingDescriptors.add(line);
- }
- }
- }
- br.close();
- this.logger.fine("Finished reading file "
- + this.missingDescriptorsFile.getAbsolutePath() + ".");
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Failed reading file "
- + this.missingDescriptorsFile.getAbsolutePath()
- + "! This means that we might forget to dowload descriptors "
- + "we are missing.", e);
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed reading file "
- + this.missingDescriptorsFile.getAbsolutePath()
- + "! This means that we might forget to dowload descriptors "
- + "we are missing.", e);
- }
- }
- // add current consensus and votes to list
- SimpleDateFormat consensusFormat =
- new SimpleDateFormat("yyyy-MM-dd HH");
- consensusFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String nowConsensusFormat = consensusFormat.format(new Date())
- + ":00:00";
- long nowConsensus = (System.currentTimeMillis() / (60L * 60L * 1000L))
- * (60L * 60L * 1000L);
- for (String authority : this.v3DirectoryAuthorities) {
- File voteFile = new File("directory-archive/vote/"
- + consensusVoteFormat.format(new Date(nowConsensus))
- + "-vote-" + authority);
- if (!this.missingDescriptors.contains("vote," + authority + ","
- + nowConsensusFormat)) {
- File voteFileDir = voteFile.getParentFile();
- String voteFileName = voteFile.getName();
- boolean voteFileFound = false;
- if (voteFileDir.exists()) {
- for (File f : Arrays.asList(voteFileDir.listFiles())) {
- if (f.getName().startsWith(voteFileName)) {
- voteFileFound = true;
- break;
- }
- }
- }
- if (!voteFileFound) {
- this.logger.finer("Adding vote to missing list: fingerprint="
- + authority + ", valid-after="
- + consensusVoteFormat.format(new Date(nowConsensus))
- + ", filename=directory-archive/vote/"
- + consensusVoteFormat.format(new Date(nowConsensus))
- + "-vote-" + authority + "-*");
- this.missingDescriptors.add("vote," + authority + ","
- + nowConsensusFormat);
- this.missingDescriptorsFileModified = true;
- }
- }
- }
- File consensusFile = new File("directory-archive/consensus/"
- + consensusVoteFormat.format(new Date(nowConsensus))
- + "-consensus");
- if (!this.missingDescriptors.contains("consensus,NA,"
- + nowConsensusFormat) && !consensusFile.exists()) {
- this.logger.finer("Adding consensus to missing list: valid-after="
- + nowConsensusFormat
- + ", filename=directory-archive/consensus/"
- + consensusVoteFormat.format(new Date(nowConsensus))
- + "-consensus");
- this.missingDescriptors.add("consensus,NA,"
- + nowConsensusFormat);
- this.missingDescriptorsFileModified = true;
- }
}
- public void store(byte[] data) throws IOException, ParseException {
- BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
- String line = br.readLine();
- if (line == null) {
- this.logger.warning("Someone gave us an empty file for storing!");
- return;
- }
- StringBuilder sb = new StringBuilder();
- sb.append(line + "\n");
- SimpleDateFormat parseFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (line.equals("network-status-version 3")) {
- // TODO when parsing the current consensus, check the fresh-until
- // time to see when we switch from hourly to half-hourly
- // consensuses; in that case, add next half-hourly consensus to
- // missing list and warn!
- boolean isConsensus = true;
- String validAfterTime = null;
- long validAfter = -1L;
- long now = System.currentTimeMillis();
- String fingerprint = null;
- SimpleDateFormat descriptorFormat =
- new SimpleDateFormat("yyyy/MM/");
- descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- while ((line = br.readLine()) != null) {
- sb.append(line + "\n");
- if (line.equals("vote-status vote")) {
- isConsensus = false;
- } else if (line.startsWith("valid-after ")) {
- validAfterTime = line.substring("valid-after ".length());
- validAfter = parseFormat.parse(validAfterTime).getTime();
- } else if (line.startsWith("dir-source ") &&
- !this.v3DirectoryAuthorities.contains(
- line.split(" ")[2]) && validAfter + 55L * 60L * 1000L > now) {
- this.logger.warning("Unknown v3 directory authority fingerprint "
- + "in consensus line '" + line + "'. You should update your "
- + "V3DirectoryAuthorities config option!");
- fingerprint = line.split(" ")[2];
- long nowConsensus = (now / (60L * 60L * 1000L))
- * (60L * 60L * 1000L);
- SimpleDateFormat consensusVoteFormat =
- new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
- consensusVoteFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File voteFile = new File("directory-archive/vote/"
- + consensusVoteFormat.format(new Date(nowConsensus))
- + "-vote-" + fingerprint);
- if (!this.missingDescriptors.contains("vote," + fingerprint
- + "," + parseFormat.format(new Date(nowConsensus)))) {
- File voteFileDir = voteFile.getParentFile();
- String voteFileName = voteFile.getName();
- boolean voteFileFound = false;
- if (voteFileDir.exists()) {
- for (File f : Arrays.asList(voteFileDir.listFiles())) {
- if (f.getName().startsWith(voteFileName)) {
- voteFileFound = true;
- break;
- }
- }
- }
- if (!voteFileFound) {
- this.logger.finer("Adding vote to missing list: "
- + "fingerprint=" + fingerprint + ", valid-after="
- + parseFormat.format(new Date(nowConsensus))
- + ", filename=directory-archive/vote/"
- + consensusVoteFormat.format(new Date(nowConsensus))
- + "-vote-" + fingerprint + "-*");
- this.missingDescriptors.add("vote," + fingerprint + ","
- + parseFormat.format(new Date(nowConsensus)));
- this.missingDescriptorsFileModified = true;
- }
- }
- } else if (line.startsWith("fingerprint ")) {
- fingerprint = line.split(" ")[1];
- } else if (line.startsWith("r ")) {
- String publishedTime = line.split(" ")[4] + " "
- + line.split(" ")[5];
- long published = parseFormat.parse(publishedTime).getTime();
- String serverDesc = Hex.encodeHexString(Base64.decodeBase64(
- line.split(" ")[3] + "=")).toLowerCase();
- // TODO are 24 hours okay?
- File descriptorFile = new File(
- "directory-archive/server-descriptor/"
- + descriptorFormat.format(new Date(published))
- + serverDesc.substring(0, 1) + "/"
- + serverDesc.substring(1, 2) + "/" + serverDesc);
- if (published + 24L * 60L * 60L * 1000L > now &&
- !this.missingDescriptors.contains("server," + serverDesc
- + "," + publishedTime) && !descriptorFile.exists()) {
- this.logger.finer("Adding server descriptor to missing list: "
- + "digest=" + serverDesc
- + ", filename=directory-archive/server-descriptor/"
- + descriptorFormat.format(new Date(published))
- + serverDesc.substring(0, 1) + "/"
- + serverDesc.substring(1, 2) + "/" + serverDesc);
- this.missingDescriptors.add("server," + serverDesc + ","
- + publishedTime);
- this.missingDescriptorsFileModified = true;
- }
- }
- }
- SimpleDateFormat printFormat =
- new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
- printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (isConsensus) {
- File consensusFile = new File("directory-archive/consensus/"
- + printFormat.format(new Date(validAfter)) + "-consensus");
- if (!consensusFile.exists()) {
- this.logger.finer("Storing consensus: valid-after="
- + validAfterTime + ", filename=directory-archive/consensus/"
- + printFormat.format(new Date(validAfter)) + "-consensus");
- consensusFile.getParentFile().mkdirs();
- BufferedOutputStream bos = new BufferedOutputStream(
- new FileOutputStream(consensusFile));
- bos.write(data, 0, data.length);
- bos.close();
- this.logger.finer("Removing consensus from missing list: "
- + "valid-after=" + validAfterTime
- + ", filename=directory-archive/consensus/"
- + printFormat.format(new Date(validAfter)) + "-consensus");
- this.missingDescriptors.remove("consensus,NA,"
- + validAfterTime);
- this.missingDescriptorsFileModified = true;
- } else {
- this.logger.finer("Not storing consensus, because we already "
- + "have it: valid-after=" + validAfterTime
- + ", filename=directory-archive/consensus/"
- + printFormat.format(new Date(validAfter)) + "-consensus");
- }
- } else {
- String ascii = new String(data, "US-ASCII");
- String startToken = "network-status-version ";
- String sigToken = "directory-signature ";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken);
- if (start < 0 || sig < 0 || sig < start) {
- this.logger.warning("Cannot determine vote digest! Skipping.");
- return;
- }
- sig += sigToken.length();
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- String digest = DigestUtils.shaHex(forDigest).toUpperCase();
- File voteFile = new File("directory-archive/vote/"
- + printFormat.format(new Date(validAfter)) + "-vote-"
- + fingerprint + "-" + digest);
- if (!voteFile.exists()) {
- this.logger.finer("Storing vote: fingerprint=" + fingerprint
- + ", valid-after="
- + printFormat.format(new Date(validAfter))
- + ", filename=directory-archive/vote/"
- + printFormat.format(new Date(validAfter)) + "-vote-"
- + fingerprint + "-" + digest);
- voteFile.getParentFile().mkdirs();
- BufferedOutputStream bos = new BufferedOutputStream(
- new FileOutputStream(voteFile));
- bos.write(data, 0, data.length);
- bos.close();
- this.logger.finer("Removing vote from missing list: "
- + "fingerprint=" + fingerprint + ", valid-after="
- + printFormat.format(new Date(validAfter))
- + ", filename=directory-archive/vote/"
- + printFormat.format(new Date(validAfter)) + "-vote-"
- + fingerprint + "-" + digest);
- this.missingDescriptors.remove("vote," + fingerprint + ","
- + validAfterTime);
- this.missingDescriptorsFileModified = true;
- } else {
- this.logger.finer("Not storing vote, because we already have "
- + "it: fingerprint=" + fingerprint + ", valid-after="
- + printFormat.format(new Date(validAfter))
- + ", filename=directory-archive/vote/"
- + printFormat.format(new Date(validAfter)) + "-vote-"
- + fingerprint + "-" + digest);
- }
- }
- } else if (line.startsWith("router ") ||
- line.startsWith("extra-info ")) {
- boolean isServerDescriptor = line.startsWith("router ");
- String publishedTime = null;
- long published = -1L;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("published ")) {
- publishedTime = line.substring("published ".length());
- published = parseFormat.parse(publishedTime).getTime();
- } else if (line.startsWith("opt extra-info-digest ") ||
- line.startsWith("extra-info-digest ")) {
- String extraInfoDigest = line.startsWith("opt ") ?
- line.split(" ")[2].toLowerCase() :
- line.split(" ")[1].toLowerCase();
- SimpleDateFormat descriptorFormat =
- new SimpleDateFormat("yyyy/MM/");
- descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File descriptorFile = new File("directory-archive/extra-info/"
- + descriptorFormat.format(new Date(published))
- + extraInfoDigest.substring(0, 1) + "/"
- + extraInfoDigest.substring(1, 2) + "/"
- + extraInfoDigest);
- if (!this.missingDescriptors.contains("extra,"
- + extraInfoDigest + "," + publishedTime) &&
- !descriptorFile.exists()) {
- this.logger.finer("Adding extra-info descriptor to missing "
- + "list: digest=" + extraInfoDigest
- + ", filename=directory-archive/extra-info/"
- + descriptorFormat.format(new Date(published))
- + extraInfoDigest.substring(0, 1) + "/"
- + extraInfoDigest.substring(1, 2) + "/"
- + extraInfoDigest);
- this.missingDescriptors.add("extra," + extraInfoDigest + ","
- + publishedTime);
- this.missingDescriptorsFileModified = true;
- }
- }
- }
- String ascii = new String(data, "US-ASCII");
- String startToken = isServerDescriptor ?
- "router " : "extra-info ";
- String sigToken = "\nrouter-signature\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- if (start < 0 || sig < 0 || sig < start) {
- this.logger.warning("Cannot determine descriptor digest! "
- + "Skipping.");
- return;
- }
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- String digest = DigestUtils.shaHex(forDigest);
- SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
- printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File descriptorFile = new File("directory-archive/"
- + (isServerDescriptor ? "server-descriptor" : "extra-info") + "/"
- + printFormat.format(new Date(published))
- + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
- + digest);
- if (!descriptorFile.exists()) {
- this.logger.finer("Storing " + (isServerDescriptor ?
- "server descriptor" : "extra-info descriptor")
- + ": digest=" + digest + ", filename=directory-archive/"
- + (isServerDescriptor ? "server-descriptor" : "extra-info")
- + "/" + printFormat.format(new Date(published))
- + digest.substring(0, 1) + "/" + digest.substring(1, 2)
- + "/" + digest);
- descriptorFile.getParentFile().mkdirs();
+
+ private void store(byte[] data, String filename) {
+ try {
+ File file = new File(filename);
+ if (!file.exists()) {
+ this.logger.finer("Storing " + filename);
+ file.getParentFile().mkdirs();
BufferedOutputStream bos = new BufferedOutputStream(
- new FileOutputStream(descriptorFile));
+ new FileOutputStream(file));
bos.write(data, 0, data.length);
bos.close();
- this.logger.finer("Removing " + (isServerDescriptor ?
- "server descriptor" : "extra-info descriptor")
- + " from missing list: digest=" + digest
- + ", filename=directory-archive/"
- + (isServerDescriptor ? "server-descriptor" : "extra-info")
- + "/" + printFormat.format(new Date(published))
- + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
- + digest);
- if (isServerDescriptor) {
- this.missingDescriptors.remove("server," + digest + ","
- + publishedTime);
- } else {
- this.missingDescriptors.remove("extra," + digest + ","
- + publishedTime);
- }
- this.missingDescriptorsFileModified = true;
- } else {
- this.logger.finer("Not storing " + (isServerDescriptor ?
- "server descriptor" : "extra-info descriptor")
- + ", because we already have it: digest=" + digest
- + ", filename=directory-archive/"
- + (isServerDescriptor ? "server-descriptor" : "extra-info")
- + "/" + printFormat.format(new Date(published))
- + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
- + digest);
}
+ } catch (IOException e) {
+ // TODO handle
}
}
- public Set<String> getMissingDescriptorUrls() {
- Set<String> urls = new HashSet<String>();
- for (String line : this.missingDescriptors) {
- if (line.startsWith("consensus,")) {
- urls.add("/tor/status-vote/current/consensus");
- } else if (line.startsWith("vote,")) {
- urls.add("/tor/status-vote/current/" + line.split(",")[1]);
- } else if (line.startsWith("server,")) {
- urls.add("/tor/server/d/" + line.split(",")[1]);
- } else if (line.startsWith("extra,")) {
- urls.add("/tor/extra/d/" + line.split(",")[1]);
- }
- }
- return urls;
+
+ public void storeConsensus(byte[] data, long validAfter) {
+ SimpleDateFormat printFormat = new SimpleDateFormat(
+ "yyyy/MM/yyyy-MM-dd-HH-mm-ss");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String filename = "directory-archive/consensus/"
+ + printFormat.format(new Date(validAfter)) + "-consensus";
+ this.store(data, filename);
}
- public void writeFile() {
- if (this.missingDescriptorsFileModified) {
- try {
- this.logger.fine("Writing file "
- + this.missingDescriptorsFile.getAbsolutePath() + "...");
- this.missingDescriptorsFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.missingDescriptorsFile));
- for (String line : this.missingDescriptors) {
- bw.write(line + "\n");
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.missingDescriptorsFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed writing "
- + this.missingDescriptorsFile.getAbsolutePath() + "!", e);
- }
- }
+
+ public void storeVote(byte[] data, long validAfter,
+ String fingerprint, String digest) {
+ SimpleDateFormat printFormat = new SimpleDateFormat(
+ "yyyy/MM/yyyy-MM-dd-HH-mm-ss");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String filename = "directory-archive/vote/"
+ + printFormat.format(new Date(validAfter)) + "-vote-"
+ + fingerprint + "-" + digest;
+ this.store(data, filename);
+ }
+
+ public void storeServerDescriptor(byte[] data, String digest,
+ long published) {
+ SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String filename = "directory-archive/server-descriptor/"
+ + printFormat.format(new Date(published))
+ + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
+ + digest;
+ this.store(data, filename);
+ }
+
+ public void storeExtraInfoDescriptor(byte[] data,
+ String extraInfoDigest, long published) {
+ SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
+ descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String filename = "directory-archive/extra-info/"
+ + descriptorFormat.format(new Date(published))
+ + extraInfoDigest.substring(0, 1) + "/"
+ + extraInfoDigest.substring(1, 2) + "/"
+ + extraInfoDigest;
+ this.store(data, filename);
}
}
diff --git a/src/CachedRelayDescriptorReader.java b/src/CachedRelayDescriptorReader.java
index 28a979c..89fa3fe 100644
--- a/src/CachedRelayDescriptorReader.java
+++ b/src/CachedRelayDescriptorReader.java
@@ -1,15 +1,12 @@
import java.io.*;
-import java.text.*;
import java.util.logging.*;
-//import org.apache.commons.codec.digest.*; TODO currently unused
/**
* Parses all descriptors in local directory cacheddesc/ and sorts them
* into directory structure in directory-archive/.
*/
public class CachedRelayDescriptorReader {
- public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
- ArchiveWriter aw) {
+ public CachedRelayDescriptorReader(RelayDescriptorParser rdp) {
// TODO check if files are stale; print out warning that Tor process
// might have died
Logger logger = Logger.getLogger(
@@ -32,9 +29,6 @@ public class CachedRelayDescriptorReader {
bis.close();
byte[] allData = baos.toByteArray();
if (f.getName().equals("cached-consensus")) {
- if (aw != null) {
- aw.store(allData);
- }
if (rdp != null) {
rdp.parse(allData);
}
@@ -62,16 +56,8 @@ public class CachedRelayDescriptorReader {
break;
}
end += endToken.length();
- /* String desc = ascii.substring(start, end);
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(allData, start, forDigest, 0, sig - start);
- String digest = DigestUtils.shaHex(forDigest);
- TODO this stuff is unused? */
byte[] descBytes = new byte[end - start];
System.arraycopy(allData, start, descBytes, 0, end - start);
- if (aw != null) {
- aw.store(descBytes);
- }
if (rdp != null) {
rdp.parse(descBytes);
}
@@ -81,9 +67,6 @@ public class CachedRelayDescriptorReader {
} catch (IOException e) {
logger.log(Level.WARNING, "Failed reading cacheddesc/ "
+ "directory.", e);
- } catch (ParseException e) {
- logger.log(Level.WARNING, "Failed reading cacheddesc/ "
- + "directory.", e);
}
}
}
diff --git a/src/Configuration.java b/src/Configuration.java
index 4ba3044..cf8c7d1 100644
--- a/src/Configuration.java
+++ b/src/Configuration.java
@@ -23,16 +23,6 @@ public class Configuration {
private List<String> relayPlatforms = new ArrayList<String>(Arrays.asList(
"Linux,Windows,Darwin,FreeBSD".split(",")));
private boolean writeDirectoryArchives = false;
- private SortedSet<String> v3DirectoryAuthorities = new TreeSet<String>(
- Arrays.asList(("14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,"
- + "E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,"
- + "D586D18309DED4CD6D57C18FDB97EFA96D330566,"
- + "585769C78764D58426B8B52B6651A5A71137189A,"
- + "27B6B5996C426270A5C95488AA5BCEB6BCC86956,"
- + "80550987E1D626E3EBA5E5E75A458DE0626D088C,"
- + "ED03BB616EB2F60BEC80151114BB25CEF515B226,"
- + "81349FC1F2DBA2C2C11B45CB9706637D480AB913,"
- + "E2A2AF570166665D738736D0DD58169CC61D8A8B").split(",")));
private boolean importCachedRelayDescriptors = true;
private boolean importDirectoryArchives = true;
private boolean importSanitizedBridges = true;
@@ -91,9 +81,6 @@ public class Configuration {
} else if (line.startsWith("WriteDirectoryArchives")) {
this.writeDirectoryArchives = Integer.parseInt(
line.split(" ")[1]) != 0;
- } else if (line.startsWith("V3DirectoryAuthorities")) {
- this.v3DirectoryAuthorities = new TreeSet<String>(
- Arrays.asList(line.split(" ")[1].split(",")));
} else if (line.startsWith("ImportCachedRelayDescriptors")) {
this.importCachedRelayDescriptors = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -191,9 +178,6 @@ public class Configuration {
public boolean getWriteDirectoryArchives() {
return this.writeDirectoryArchives;
}
- public SortedSet<String> getV3DirectoryAuthorities() {
- return this.v3DirectoryAuthorities;
- }
public boolean getImportCachedRelayDescriptors() {
return this.importCachedRelayDescriptors;
}
diff --git a/src/Main.java b/src/Main.java
index 2d96396..fa40fa0 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -38,40 +38,50 @@ public class Main {
new ServerDescriptorStatsFileHandler(config.getRelayVersions(),
config.getRelayPlatforms()) : null;
- // Prepare relay descriptor parser (only if we are writing the
- // stats)
+ // Prepare writing relay descriptor archive to disk
+ ArchiveWriter aw = config.getWriteDirectoryArchives() ?
+ new ArchiveWriter() : null;
+
+ // Prepare relay descriptor parser (only if we are writing stats or
+ // directory archives to disk)
RelayDescriptorParser rdp = config.getWriteConsensusStats() ||
config.getWriteBridgeStats() || config.getWriteDirreqStats() ||
- config.getWriteServerDescriptorStats() ?
- new RelayDescriptorParser(csfh, bsfh, dsfh, sdsfh, countries,
+ config.getWriteServerDescriptorStats() ||
+ config.getWriteDirectoryArchives() ?
+ new RelayDescriptorParser(csfh, bsfh, dsfh, sdsfh, aw, countries,
directories) : null;
- // Prepare writing relay descriptor archive to disk
- ArchiveWriter aw = config.getWriteDirectoryArchives() ?
- new ArchiveWriter(config.getV3DirectoryAuthorities()) : null;
-
// Import/download relay descriptors from the various sources
- if (config.getImportCachedRelayDescriptors()) {
- new CachedRelayDescriptorReader(rdp, aw);
- }
- if (config.getImportDirectoryArchives()) {
- new ArchiveReader(rdp, "archives");
- }
- if (config.getDownloadRelayDescriptors()) {
- new RelayDescriptorDownloader(rdp, aw,
- config.getDownloadFromDirectoryAuthorities(),
- directories);
+ if (rdp != null) {
+ RelayDescriptorDownloader rdd = null;
+ if (config.getDownloadRelayDescriptors()) {
+ List<String> dirSources =
+ config.getDownloadFromDirectoryAuthorities();
+ boolean downloadCurrentConsensus = aw != null || csfh != null ||
+ bsfh != null || sdsfh != null;
+ boolean downloadCurrentVotes = aw != null;
+ boolean downloadAllServerDescriptors = aw != null || sdsfh != null;
+ boolean downloadAllExtraInfos = aw != null;
+ Set<String> downloadDescriptorsForRelays = directories;
+ rdd = new RelayDescriptorDownloader(rdp, dirSources,
+ downloadCurrentConsensus, downloadCurrentVotes,
+ downloadAllServerDescriptors, downloadAllExtraInfos,
+ downloadDescriptorsForRelays);
+ rdp.setRelayDescriptorDownloader(rdd);
+ }
+ if (config.getImportCachedRelayDescriptors()) {
+ new CachedRelayDescriptorReader(rdp);
+ }
+ if (config.getImportDirectoryArchives()) {
+ new ArchiveReader(rdp, "archives");
+ }
+ if (rdd != null) {
+ rdd.downloadMissingDescriptors();
+ rdd.writeFile();
+ }
}
// Write output to disk that only depends on relay descriptors
- if (aw != null) {
- aw.writeFile();
- aw = null;
- }
- if (rdp != null) {
- rdp.writeFile();
- rdp = null;
- }
if (dsfh != null) {
dsfh.writeFile();
dsfh = null;
diff --git a/src/RelayDescriptorDownloader.java b/src/RelayDescriptorDownloader.java
index a3bc7d3..d7654c5 100644
--- a/src/RelayDescriptorDownloader.java
+++ b/src/RelayDescriptorDownloader.java
@@ -1,38 +1,397 @@
import java.io.*;
import java.net.*;
+import java.text.*;
import java.util.*;
import java.util.logging.*;
-import org.apache.commons.codec.digest.*;
/**
- * Download the current consensus and relevant extra-info descriptors and
- * hand them to the relay descriptor parser.
+ * Downloads missing relay descriptors from the directories via HTTP.
+ * Keeps a list of missing descriptors that gets updated by parse results
+ * from <code>RelayDescriptorParser</code>. Only descriptors on that
+ * missing list that we think might be available on the directories are
+ * downloaded.
*/
public class RelayDescriptorDownloader {
+
+ /**
+ * Text file containing the descriptors that we are missing and that we
+ * want to download in <code>downloadMissingDescriptors</code>.
+ * Lines are formatted as:
+ * - "consensus,<validafter>,<parsed>",
+ * - "vote,<validafter>,<fingerprint>,<parsed>",
+ * - "server,<published>,<relayid>,<descid>,<parsed>", or
+ * - "extra,<published>,<relayid>,<descid><parsed>".
+ */
+ private File missingDescriptorsFile;
+
+ /**
+ * Relay descriptors that we are missing and that we want to download
+ * either in this execution or write to disk and try next time. Map keys
+ * contain comma-separated values as in the missing descriptors files
+ * without the parsed column. Map values contain the parsed column.
+ */
+ private SortedMap<String, String> missingDescriptors;
+
+ /**
+ * <code>RelayDescriptorParser</code> that we will hand over the
+ * downloaded descriptors for parsing.
+ */
+ private RelayDescriptorParser rdp;
+
+ /**
+ * Directories that we will try to download missing descriptors from.
+ */
+ private List<String> dirSources;
+
+ /**
+ * Should we try to download the current consensus if we don't have it?
+ */
+ private boolean downloadCurrentConsensus;
+
+ /**
+ * Should we try to download current votes if we don't have them?
+ */
+ private boolean downloadCurrentVotes;
+
+ /**
+ * Should we try to download all missing server descriptors that have
+ * been published within the past 24 hours?
+ */
+ private boolean downloadAllServerDescriptors;
+
+ /**
+ * Should we try to download all missing extra-info descriptors that
+ * have been published within the past 24 hours?
+ */
+ private boolean downloadAllExtraInfos;
+
+ /**
+ * Should we try to download missing server and extra-info descriptors
+ * of certain relays that have been published within the past 24 hours?
+ */
+ private Set<String> downloadDescriptorsForRelays;
+
+ /**
+ * valid-after time that we expect the current consensus and votes to
+ * have, formatted "yyyy-MM-dd HH:mm:ss". We only expect to find
+ * consensuses and votes with this valid-after time on the directories.
+ * This time is initialized as the beginning of the current hour.
+ */
+ private String currentValidAfter;
+
+ /**
+ * Cut-off time for missing server and extra-info descriptors, formatted
+ * "yyyy-MM-dd HH:mm:ss". This time is initialized as the current system
+ * time minus 24 hours.
+ */
+ private String descriptorCutOff;
+
+ /**
+ * Current timestamp that is written to the missing list for descriptors
+ * that we parsed in this execution. This timestamp is most useful for
+ * debugging purposes when looking at the missing list. For execution it
+ * only matters whether the parsed time is "NA" or has some other value.
+ */
+ private String parsedTimestampString;
+
+ /**
+ * Logger for this class.
+ */
+ private Logger logger;
+
+ /**
+ * Initializes this class, including reading in missing descriptors from
+ * <code>stats/missing-relay-descriptors</code>.
+ */
public RelayDescriptorDownloader(RelayDescriptorParser rdp,
- ArchiveWriter aw, List<String> authorities,
- SortedSet<String> directories) {
- Logger logger = Logger.getLogger(
- RelayDescriptorDownloader.class.getName());
- List<String> remainingAuthorities =
- new ArrayList<String>(authorities);
- Set<String> urls = new HashSet<String>();
- Set<String> downloaded = new HashSet<String>();
- if (rdp != null) {
- urls.addAll(rdp.getMissingDescriptorUrls());
+ List<String> dirSources, boolean downloadCurrentConsensus,
+ boolean downloadCurrentVotes, boolean downloadAllServerDescriptors,
+ boolean downloadAllExtraInfos,
+ Set<String> downloadDescriptorsForRelays) {
+
+ /* Memorize argument values. */
+ this.rdp = rdp;
+ this.dirSources = dirSources;
+ this.downloadCurrentConsensus = downloadCurrentConsensus;
+ this.downloadCurrentVotes = downloadCurrentVotes;
+ this.downloadAllServerDescriptors = downloadAllServerDescriptors;
+ this.downloadAllExtraInfos = downloadAllExtraInfos;
+ this.downloadDescriptorsForRelays = downloadDescriptorsForRelays;
+
+ /* Initialize logger. */
+ this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
+
+ /* Prepare cut-off times and timestamp for missing descriptors
+ * list. */
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long now = System.currentTimeMillis();
+ this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) *
+ (60L * 60L * 1000L));
+ this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L);
+ this.parsedTimestampString = format.format(now);
+
+ /* Initialize missing list and put current consensus on it if we want
+ * it. */
+ this.missingDescriptors = new TreeMap<String, String>();
+ if (this.downloadCurrentConsensus) {
+ this.missingDescriptors.put("consensus," + this.currentValidAfter,
+ "NA");
+ }
+
+ /* Read list of missing descriptors from disk and memorize those that
+ * we are interested in and that are likely to be found on the
+ * directory servers. */
+ this.missingDescriptorsFile = new File(
+ "stats/missing-relay-descriptors");
+ if (this.missingDescriptorsFile.exists()) {
+ try {
+ this.logger.fine("Reading file "
+ + this.missingDescriptorsFile.getAbsolutePath() + "...");
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.missingDescriptorsFile));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ if (line.split(",").length > 2) {
+ String published = line.split(",")[1];
+ if (((line.startsWith("consensus,") ||
+ line.startsWith("vote,")) &&
+ this.currentValidAfter.equals(published)) ||
+ ((line.startsWith("server,") ||
+ line.startsWith("extra,")) &&
+ this.descriptorCutOff.compareTo(published) <= 0)) {
+ int separateAt = line.lastIndexOf(",");
+ this.missingDescriptors.put(line.substring(0,
+ separateAt), line.substring(separateAt + 1));
+ }
+ } else {
+ this.logger.fine("Invalid line '" + line + "' in "
+ + this.missingDescriptorsFile.getAbsolutePath()
+ + ". Ignoring.");
+ }
+ }
+ br.close();
+ this.logger.fine("Finished reading file "
+ + this.missingDescriptorsFile.getAbsolutePath() + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed to read file "
+ + this.missingDescriptorsFile.getAbsolutePath()
+ + "! This means that we might forget to dowload relay "
+ + "descriptors we are missing.", e);
+ }
+ }
+ }
+
+ /**
+ * We have parsed a consensus. Take this consensus off the missing list
+ * and add the votes created by the given <code>dirSources</code> and
+ * the <code>serverDescriptors</code> in the format
+ * "<published>,<relayid>,<descid>" to that list.
+ */
+ public void haveParsedConsensus(String validAfter,
+ Set<String> dirSources, Set<String> serverDescriptors) {
+
+ /* Mark consensus as parsed. */
+ if (this.currentValidAfter.equals(validAfter)) {
+ String consensusKey = "consensus," + validAfter;
+ this.missingDescriptors.put(consensusKey,
+ this.parsedTimestampString);
+
+ /* Add votes to missing list. */
+ if (this.downloadCurrentVotes) {
+ for (String dirSource : dirSources) {
+ String voteKey = "vote," + validAfter + "," + dirSource;
+ if (!this.missingDescriptors.containsKey(voteKey)) {
+ this.missingDescriptors.put(voteKey, "NA");
+ }
+ }
+ }
+ }
+
+ /* Add server descriptors to missing list. */
+ if (this.downloadAllServerDescriptors ||
+ this.downloadDescriptorsForRelays != null) {
+ for (String serverDescriptor : serverDescriptors) {
+ String published = serverDescriptor.split(",")[0];
+ if (this.descriptorCutOff.compareTo(published) <= 0) {
+ if (this.downloadAllServerDescriptors ||
+ (this.downloadDescriptorsForRelays != null &&
+ this.downloadDescriptorsForRelays.contains(
+ serverDescriptor.split(",")[1].toUpperCase()))) {
+ String serverDescriptorKey = "server," + serverDescriptor;
+ if (!this.missingDescriptors.containsKey(
+ serverDescriptorKey)) {
+ this.missingDescriptors.put(serverDescriptorKey, "NA");
+ }
+ }
+ }
+ }
}
+ }
+
+ /**
+ * We have parsed a vote. Take this vote off the missing list.
+ */
+ public void haveParsedVote(String validAfter, String fingerprint,
+ Set<String> serverDescriptors) {
+
+ /* Mark consensus as parsed. */
+ if (this.currentValidAfter.equals(validAfter)) {
+ String voteKey = "vote," + validAfter + "," + fingerprint;
+ this.missingDescriptors.put(voteKey, this.parsedTimestampString);
+ }
+
+ /* Add server descriptors to missing list. */
+ if (this.downloadAllServerDescriptors ||
+ this.downloadDescriptorsForRelays != null) {
+ for (String serverDescriptor : serverDescriptors) {
+ String published = serverDescriptor.split(",")[0];
+ if (this.descriptorCutOff.compareTo(published) < 0) {
+ if (this.downloadDescriptorsForRelays == null ||
+ this.downloadDescriptorsForRelays.contains(
+ serverDescriptor.split(",")[1].toUpperCase())) {
+ String serverDescriptorKey = "server," + serverDescriptor;
+ if (!this.missingDescriptors.containsKey(
+ serverDescriptorKey)) {
+ this.missingDescriptors.put(serverDescriptorKey, "NA");
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * We have parsed a server descriptor. Take this server descriptor off
+ * the missing list and put the extra-info descriptor digest on that
+ * list.
+ */
+ public void haveParsedServerDescriptor(String published,
+ String relayIdentity, String serverDescriptorDigest,
+ String extraInfoDigest) {
+
+ /* Mark server descriptor as parsed. */
+ if (this.descriptorCutOff.compareTo(published) <= 0) {
+ String serverDescriptorKey = "server," + published + ","
+ + relayIdentity + "," + serverDescriptorDigest;
+ this.missingDescriptors.put(serverDescriptorKey,
+ this.parsedTimestampString);
+
+ /* Add extra-info descriptor to missing list. */
+ if (extraInfoDigest != null && (this.downloadAllExtraInfos ||
+ (this.downloadDescriptorsForRelays != null &&
+ this.downloadDescriptorsForRelays.contains(relayIdentity.
+ toUpperCase())))) {
+ String extraInfoKey = "extra," + published + ","
+ + relayIdentity + "," + extraInfoDigest;
+ if (!this.missingDescriptors.containsKey(extraInfoKey)) {
+ this.missingDescriptors.put(extraInfoKey, "NA");
+ }
+ }
+ }
+ }
+
+ /**
+ * We have parsed an extra-info descriptor. Take it off the missing
+ * list.
+ */
+ public void haveParsedExtraInfoDescriptor(String published,
+ String relayIdentity, String extraInfoDigest) {
+ if (this.descriptorCutOff.compareTo(published) <= 0) {
+ String extraInfoKey = "extra," + published + ","
+ + relayIdentity + "," + extraInfoDigest;
+ this.missingDescriptors.put(extraInfoKey,
+ this.parsedTimestampString);
+ }
+ }
+
+ /**
+ * Downloads missing descriptors that we think might still be available
+ * on the directories.
+ */
+ public void downloadMissingDescriptors() {
+
+ /* Update cut-off times to reflect that execution so far might have
+ * taken a few minutes and that some descriptors aren't available on
+ * the directories anymore. */
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long now = System.currentTimeMillis();
+ this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) *
+ (60L * 60L * 1000L));
+ this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L);
+
+ /* Remember which directories remain as source for downloading
+ * descriptors. */
+ List<String> remainingDirSources =
+ new ArrayList<String>(this.dirSources);
+
+ /* URLs of descriptors we want to download. */
+ SortedSet<String> urls = new TreeSet<String>();
+
+ /* URLs of descriptors we have downloaded or at least tried to
+ * download. */
+ SortedSet<String> downloaded = new TreeSet<String>();
+
+ /* We might need more than one iteration for downloading descriptors,
+ * because we might learn about descriptors while parsing those that
+ * we got. In every iteration, compile a new list of URLs, remove
+ * those that we tried before, and download the remaining ones. Stop
+ * when there are no new URLs anymore. */
do {
- if (aw != null) {
- urls.addAll(aw.getMissingDescriptorUrls());
+
+ /* Compile list of URLs to download in this iteration. */
+ urls.clear();
+ for (Map.Entry<String, String> e :
+ this.missingDescriptors.entrySet()) {
+ if (e.getValue().equals("NA")) {
+ String[] parts = e.getKey().split(",");
+ if (parts[0].equals("consensus") &&
+ this.downloadCurrentConsensus &&
+ this.currentValidAfter.equals(parts[1])) {
+ urls.add("/tor/status-vote/current/consensus");
+ } else if (parts[0].equals("vote") &&
+ this.downloadCurrentVotes &&
+ this.currentValidAfter.equals(parts[1])) {
+ urls.add("/tor/status-vote/current/" + parts[2]);
+ } else if (parts[0].equals("server") &&
+ (this.downloadAllServerDescriptors ||
+ (this.downloadDescriptorsForRelays != null &&
+ this.downloadDescriptorsForRelays.contains(parts[2].
+ toUpperCase()))) &&
+ this.descriptorCutOff.compareTo(parts[1]) <= 0) {
+ urls.add("/tor/server/d/" + parts[3]);
+ } else if (parts[0].equals("extra") &&
+ (this.downloadAllExtraInfos ||
+ (this.downloadDescriptorsForRelays != null &&
+ this.downloadDescriptorsForRelays.contains(parts[2].
+ toUpperCase()))) &&
+ this.descriptorCutOff.compareTo(parts[1]) <= 0) {
+ urls.add("/tor/extra/d/" + parts[3]);
+ }
+ }
}
urls.removeAll(downloaded);
- SortedSet<String> sortedAuthorities =
- new TreeSet<String>(remainingAuthorities);
- SortedSet<String> sortedUrls = new TreeSet<String>(urls);
+
+ /* Log what we're downloading. */
+ StringBuilder sb = new StringBuilder("Downloading " + urls.size()
+ + " descriptors:");
+ for (String url : urls) {
+ sb.append(url + "\n");
+ }
+ this.logger.fine(sb.toString());
+
+ /* We are trying to download these descriptors from each directory
+ * source one after the other until we got it from one. For each
+ * directory source we are removing the URLs from urls and putting
+ * those the we want to retry into retryUrls. Once we are done, we
+ * move the URLs back to urls and try the next directory source. */
+ SortedSet<String> currentDirSources =
+ new TreeSet<String>(remainingDirSources);
SortedSet<String> retryUrls = new TreeSet<String>();
- while (!sortedAuthorities.isEmpty() && !sortedUrls.isEmpty()) {
- String authority = sortedAuthorities.first();
- String url = sortedUrls.first();
+ while (!currentDirSources.isEmpty() && !urls.isEmpty()) {
+ String authority = currentDirSources.first();
+ String url = urls.first();
try {
URL u = new URL("http://"; + authority + url);
HttpURLConnection huc =
@@ -40,7 +399,7 @@ public class RelayDescriptorDownloader {
huc.setRequestMethod("GET");
huc.connect();
int response = huc.getResponseCode();
- logger.fine("Downloading http://"; + authority + url + " -> "
+ logger.finer("Downloading http://"; + authority + url + " -> "
+ response);
if (response == 200) {
BufferedInputStream in = new BufferedInputStream(
@@ -49,65 +408,25 @@ public class RelayDescriptorDownloader {
int len;
byte[] data = new byte[1024];
while ((len = in.read(data, 0, 1024)) >= 0) {
- // we need to write the result to a byte array in order
- // to get a sane digest; otherwise, descriptors with
- // non-ASCII chars lead to different digests.
baos.write(data, 0, len);
}
in.close();
- String digest = null;
byte[] allData = baos.toByteArray();
- int beforeSig = new String(allData).indexOf(
- "\nrouter-signature\n")
- + "\nrouter-signature\n".length();
- byte[] noSig = new byte[beforeSig];
- System.arraycopy(allData, 0, noSig, 0, beforeSig);
- digest = DigestUtils.shaHex(noSig);
- // TODO UTF-8 may be wrong, but we don't care about the fields
- // containing non-ASCII
- // String result = new String(allData, "UTF-8"); TODO
- boolean verified = false;
- if (url.contains("/tor/server/d/") ||
- url.contains("/tor/extra/d/")) {
- if (url.endsWith(digest)) {
- verified = true;
- } else {
- logger.warning("Downloaded descriptor digest (" + digest
- + " doesn't match what we asked for (" + url + ")! "
- + "Retrying.");
- retryUrls.add(url);
- }
- } else {
- verified = true;
- // TODO verify downloaded consensuses and votes, too
- }
- if (verified) {
- if (rdp != null) {
- rdp.parse(allData);
- }
- if (aw != null) {
- try {
- aw.store(allData);
- } catch (Exception e) {
- e.printStackTrace();
- //TODO find better way to handle this
- }
- }
- }
+ rdp.parse(allData);
} else {
retryUrls.add(url);
}
- sortedUrls.remove(url);
- if (sortedUrls.isEmpty()) {
- sortedAuthorities.remove(authority);
- sortedUrls.addAll(retryUrls);
+ urls.remove(url);
+ if (urls.isEmpty()) {
+ currentDirSources.remove(authority);
+ urls.addAll(retryUrls);
retryUrls.clear();
}
} catch (IOException e) {
- remainingAuthorities.remove(authority);
- sortedAuthorities.remove(authority);
- if (!remainingAuthorities.isEmpty()) {
- logger.log(Level.INFO, "Failed downloading from "
+ remainingDirSources.remove(authority);
+ currentDirSources.remove(authority);
+ if (!remainingDirSources.isEmpty()) {
+ logger.log(Level.FINE, "Failed downloading from "
+ authority + "!", e);
} else {
logger.log(Level.WARNING, "Failed downloading from "
@@ -119,5 +438,24 @@ public class RelayDescriptorDownloader {
downloaded.addAll(urls);
} while (!urls.isEmpty());
}
-}
+ public void writeFile() {
+ try {
+ this.logger.fine("Writing file "
+ + this.missingDescriptorsFile.getAbsolutePath() + "...");
+ this.missingDescriptorsFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.missingDescriptorsFile));
+ for (Map.Entry<String, String> e :
+ this.missingDescriptors.entrySet()) {
+ bw.write(e.getKey() + "," + e.getValue() + "\n");
+ }
+ bw.close();
+ this.logger.fine("Finished writing file "
+ + this.missingDescriptorsFile.getAbsolutePath() + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed writing "
+ + this.missingDescriptorsFile.getAbsolutePath() + "!", e);
+ }
+ }
+}
diff --git a/src/RelayDescriptorParser.java b/src/RelayDescriptorParser.java
index 81661ef..b7af740 100644
--- a/src/RelayDescriptorParser.java
+++ b/src/RelayDescriptorParser.java
@@ -6,252 +6,304 @@ import org.apache.commons.codec.digest.*;
import org.apache.commons.codec.binary.*;
/**
- * Parse the contents of a network status consensus and pass on the
- * relevant contents to the stats file handlers.
+ * Parses relay descriptors including network status consensuses and
+ * votes, server and extra-info descriptors, and passes the results to the
+ * stats handlers, to the archive writer, or to the relay descriptor
+ * downloader.
*/
public class RelayDescriptorParser {
- private File relayDescriptorParseHistoryFile;
- private SortedMap<String, String> lastParsedExtraInfos;
- private String lastParsedConsensus;
- private boolean relayDescriptorParseHistoryModified = false;
+
+ /**
+ * Stats file handler that accepts parse results for directory request
+ * statistics.
+ */
private DirreqStatsFileHandler dsfh;
+
+ /**
+ * Stats file handler that accepts parse results for consensus
+ * statistics.
+ */
private ConsensusStatsFileHandler csfh;
+
+ /**
+ * Stats file handler that accepts parse results for bridge statistics.
+ */
private BridgeStatsFileHandler bsfh;
+
+ /**
+ * Stats file handler that accepts parse results for server descriptor
+ * statistics.
+ */
private ServerDescriptorStatsFileHandler sdsfh;
+
+ /**
+ * File writer that writes descriptor contents to files in a
+ * directory-archive directory structure.
+ */
+ private ArchiveWriter aw;
+
+ /**
+ * Missing descriptor downloader that uses the parse results to learn
+ * which descriptors we are missing and want to download.
+ */
+ private RelayDescriptorDownloader rdd;
+
+ /**
+ * Countries that we care about for directory request and bridge
+ * statistics.
+ */
private SortedSet<String> countries;
+
+ /**
+ * Directories that we care about for directory request statistics.
+ */
private SortedSet<String> directories;
+
+ /**
+ * Logger for this class.
+ */
private Logger logger;
+
+ /**
+ * Initializes this class.
+ */
public RelayDescriptorParser(ConsensusStatsFileHandler csfh,
BridgeStatsFileHandler bsfh, DirreqStatsFileHandler dsfh,
- ServerDescriptorStatsFileHandler sdsfh, SortedSet<String> countries,
- SortedSet<String> directories) {
- this.relayDescriptorParseHistoryFile = new File(
- "stats/relay-descriptor-parse-history");
+ ServerDescriptorStatsFileHandler sdsfh, ArchiveWriter aw,
+ SortedSet<String> countries, SortedSet<String> directories) {
this.csfh = csfh;
this.bsfh = bsfh;
this.dsfh = dsfh;
this.sdsfh = sdsfh;
+ this.aw = aw;
this.countries = countries;
this.directories = directories;
- this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
- this.lastParsedConsensus = null;
- this.lastParsedExtraInfos = new TreeMap<String, String>();
- if (this.relayDescriptorParseHistoryFile.exists()) {
- this.logger.fine("Reading file "
- + this.relayDescriptorParseHistoryFile.getAbsolutePath()
- + "...");
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- this.relayDescriptorParseHistoryFile));
- String line = null;
+ }
+
+ public void setRelayDescriptorDownloader(
+ RelayDescriptorDownloader rdd) {
+ this.rdd = rdd;
+ }
+
+ public void parse(byte[] data) {
+ try {
+ /* Convert descriptor to ASCII for parsing. This means we'll lose
+ * the non-ASCII chars, but we don't care about them for parsing
+ * anyway. */
+ BufferedReader br = new BufferedReader(new StringReader(new String(
+ data, "US-ASCII")));
+ String line = br.readLine();
+ if (line == null) {
+ this.logger.fine("We were given an empty descriptor for "
+ + "parsing. Ignoring.");
+ return;
+ }
+ SimpleDateFormat parseFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ if (line.equals("network-status-version 3")) {
+ // TODO when parsing the current consensus, check the fresh-until
+ // time to see when we switch from hourly to half-hourly
+ // consensuses
+ boolean isConsensus = true;
+ int exit = 0, fast = 0, guard = 0, running = 0, stable = 0;
+ String validAfterTime = null, descriptorIdentity = null;
+ StringBuilder descriptorIdentities = new StringBuilder();
+ String fingerprint = null;
+ long validAfter = -1L;
+ SortedSet<String> dirSources = new TreeSet<String>();
+ SortedSet<String> serverDescriptors = new TreeSet<String>();
+ SortedSet<String> hashedRelayIdentities = new TreeSet<String>();
while ((line = br.readLine()) != null) {
- if (line.startsWith("consensus")) {
- this.lastParsedConsensus = line.split(",")[2];
- } else if (line.startsWith("extrainfo")) {
- this.lastParsedExtraInfos.put(line.split(",")[1],
- line.split(",")[2]);
+ if (line.equals("vote-status vote")) {
+ isConsensus = false;
+ } else if (line.startsWith("valid-after ")) {
+ validAfterTime = line.substring("valid-after ".length());
+ validAfter = parseFormat.parse(validAfterTime).getTime();
+ } else if (line.startsWith("dir-source ")) {
+ dirSources.add(line.split(" ")[2]);
+ } else if (line.startsWith("fingerprint ")) {
+ fingerprint = line.split(" ")[1];
+ } else if (line.startsWith("r ")) {
+ String publishedTime = line.split(" ")[4] + " "
+ + line.split(" ")[5];
+ String relayIdentity = Hex.encodeHexString(
+ Base64.decodeBase64(line.split(" ")[2] + "=")).
+ toLowerCase();
+ String serverDesc = Hex.encodeHexString(Base64.decodeBase64(
+ line.split(" ")[3] + "=")).toLowerCase();
+ serverDescriptors.add(publishedTime + "," + relayIdentity
+ + "," + serverDesc);
+ hashedRelayIdentities.add(DigestUtils.shaHex(
+ Base64.decodeBase64(relayIdentity + "=")).toUpperCase());
+ descriptorIdentity = line.split(" ")[3];
+ } else if (line.startsWith("s ")) {
+ if (line.contains(" Running")) {
+ exit += line.contains(" Exit") ? 1 : 0;
+ fast += line.contains(" Fast") ? 1 : 0;
+ guard += line.contains(" Guard") ? 1 : 0;
+ stable += line.contains(" Stable") ? 1 : 0;
+ running++;
+ descriptorIdentities.append("," + descriptorIdentity);
+ }
}
}
- br.close();
- this.logger.fine("Finished reading file "
- + this.relayDescriptorParseHistoryFile.getAbsolutePath()
- + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed reading file "
- + this.relayDescriptorParseHistoryFile.getAbsolutePath()
- + "!", e);
- }
- }
- }
- public void parse(byte[] data) throws IOException {
- BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
- String line = br.readLine();
- if (line == null) {
- this.logger.warning("Parsing empty file?");
- return;
- }
- if (line.equals("network-status-version 3")) {
- int exit = 0, fast = 0, guard = 0, running = 0, stable = 0;
- String validAfter = null, rLine = null;
- StringBuilder descriptorIdentities = new StringBuilder();
- while ((line = br.readLine()) != null) {
- if (line.startsWith("valid-after ")) {
- validAfter = line.substring("valid-after ".length());
- if (this.lastParsedConsensus == null ||
- validAfter.compareTo(this.lastParsedConsensus) > 0) {
- this.lastParsedConsensus = validAfter;
- relayDescriptorParseHistoryModified = true;
- }
- } else if (line.equals("vote-status vote")) {
- return;
- } else if (line.startsWith("r ")) {
+ if (isConsensus) {
if (this.bsfh != null) {
- String hashedRelay = DigestUtils.shaHex(Base64.decodeBase64(
- line.split(" ")[2] + "=")).toUpperCase();
- this.bsfh.addHashedRelay(hashedRelay);
+ for (String hashedRelayIdentity : hashedRelayIdentities) {
+ this.bsfh.addHashedRelay(hashedRelayIdentity);
+ }
+ }
+ if (this.csfh != null) {
+ this.csfh.addConsensusResults(validAfterTime, exit, fast,
+ guard, running, stable);
}
- rLine = line;
- } else if (line.startsWith("s ")) {
- if (line.contains(" Running")) {
- exit += line.contains(" Exit") ? 1 : 0;
- fast += line.contains(" Fast") ? 1 : 0;
- guard += line.contains(" Guard") ? 1 : 0;
- stable += line.contains(" Stable") ? 1 : 0;
- running++;
- descriptorIdentities.append("," + rLine.split(" ")[3]);
+ if (this.sdsfh != null) {
+ this.sdsfh.addConsensus(validAfterTime,
+ descriptorIdentities.toString().substring(1));
+ }
+ if (this.rdd != null) {
+ this.rdd.haveParsedConsensus(validAfterTime, dirSources,
+ serverDescriptors);
+ }
+ if (this.aw != null) {
+ this.aw.storeConsensus(data, validAfter);
+ }
+ } else {
+ if (this.rdd != null) {
+ this.rdd.haveParsedVote(validAfterTime, fingerprint,
+ serverDescriptors);
+ }
+ if (this.aw != null) {
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "network-status-version ";
+ String sigToken = "directory-signature ";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken);
+ if (start >= 0 && sig >= 0 && sig > start) {
+ sig += sigToken.length();
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ String digest = DigestUtils.shaHex(forDigest).toUpperCase();
+ this.aw.storeVote(data, validAfter,
+ new ArrayList<String>(dirSources).get(0), digest);
+ }
}
}
- }
- if (this.csfh != null) {
- this.csfh.addConsensusResults(validAfter, exit, fast, guard,
- running, stable);
- }
- if (this.sdsfh != null) {
- this.sdsfh.addConsensus(validAfter,
- descriptorIdentities.toString().substring(1));
- }
- } else if (line.startsWith("router ")) {
- String platformLine = null, publishedLine = null,
- bandwidthLine = null;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("platform ")) {
- platformLine = line;
- } else if (line.startsWith("published ")) {
- publishedLine = line;
- } else if (line.startsWith("bandwidth ")) {
- bandwidthLine = line;
+ } else if (line.startsWith("router ")) {
+ String platformLine = null, publishedLine = null,
+ publishedTime = null, bandwidthLine = null,
+ extraInfoDigest = null, relayIdentifier = null;
+ long published = -1L;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("platform ")) {
+ platformLine = line;
+ } else if (line.startsWith("published ")) {
+ publishedLine = line;
+ publishedTime = line.substring("published ".length());
+ published = parseFormat.parse(publishedTime).getTime();
+ } else if (line.startsWith("opt fingerprint") ||
+ line.startsWith("fingerprint")) {
+ relayIdentifier = line.substring(line.startsWith("opt ") ?
+ "opt fingerprint".length() : "fingerprint".length()).
+ replaceAll(" ", "").toLowerCase();
+ } else if (line.startsWith("bandwidth ")) {
+ bandwidthLine = line;
+ } else if (line.startsWith("opt extra-info-digest ") ||
+ line.startsWith("extra-info-digest ")) {
+ extraInfoDigest = line.startsWith("opt ") ?
+ line.split(" ")[2].toLowerCase() :
+ line.split(" ")[1].toLowerCase();
+ }
}
- }
- String ascii = new String(data, "US-ASCII");
- String startToken = "router ";
- String sigToken = "\nrouter-signature\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- if (start < 0 || sig < 0 || sig < start) {
- this.logger.warning("Cannot determine descriptor digest! "
- + "Skipping.");
- return;
- }
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- String descriptorIdentity = Base64.encodeBase64String(
- DigestUtils.sha(forDigest)).substring(0, 27);
- if (this.sdsfh != null) {
- this.sdsfh.addServerDescriptor(descriptorIdentity, platformLine,
- publishedLine, bandwidthLine);
- }
- } else if (line.startsWith("extra-info ") && this.dsfh != null &&
- directories.contains(line.split(" ")[2])) {
- String dir = line.split(" ")[2];
- String statsEnd = null, date = null, v3ips = null;
- boolean skip = false;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("dirreq-stats-end ")) {
- statsEnd = line.split(" ")[1] + " " + line.split(" ")[2];
- date = line.split(" ")[1];
- // trusted had very strange dirreq-v3-shares here...
- skip = dir.equals("8522EB98C91496E80EC238E732594D1509158E77")
- && (date.equals("2009-09-10") || date.equals("2009-09-11"));
- } else if (line.startsWith("dirreq-v3-reqs ")
- && line.length() > "dirreq-v3-reqs ".length()) {
- v3ips = line.split(" ")[1];
- } else if (line.startsWith("dirreq-v3-share ")
- && v3ips != null && !skip) {
- Map<String, String> obs = new HashMap<String, String>();
- String[] parts = v3ips.split(",");
- for (String p : parts) {
- for (String c : this.countries) {
- if (p.startsWith(c)) {
- obs.put(c, p.substring(3));
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "router ";
+ String sigToken = "\nrouter-signature\n";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ String digest = null, descriptorIdentity = null;
+ if (start >= 0 || sig >= 0 || sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ descriptorIdentity = Base64.encodeBase64String(
+ DigestUtils.sha(forDigest)).substring(0, 27);
+ digest = DigestUtils.shaHex(forDigest);
+ }
+ if (this.aw != null && digest != null) {
+ this.aw.storeServerDescriptor(data, digest, published);
+ }
+ if (this.rdd != null && digest != null) {
+ this.rdd.haveParsedServerDescriptor(publishedTime,
+ relayIdentifier, digest, extraInfoDigest);
+ }
+ if (this.sdsfh != null && descriptorIdentity != null) {
+ this.sdsfh.addServerDescriptor(descriptorIdentity, platformLine,
+ publishedLine, bandwidthLine);
+ }
+ } else if (line.startsWith("extra-info ")) {
+ String publishedTime = null, relayIdentifier = line.split(" ")[2];
+ long published = -1L;
+ String dir = line.split(" ")[2];
+ String date = null, v3Reqs = null;
+ boolean skip = false;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("published ")) {
+ publishedTime = line.substring("published ".length());
+ published = parseFormat.parse(publishedTime).getTime();
+ } else if (line.startsWith("dirreq-stats-end ")) {
+ date = line.split(" ")[1];
+ // trusted had very strange dirreq-v3-shares here...
+ // TODO don't check that here, but in DirreqStatsFileHandler
+ skip = dir.equals("8522EB98C91496E80EC238E732594D1509158E77")
+ && (date.equals("2009-09-10") ||
+ date.equals("2009-09-11"));
+ } else if (line.startsWith("dirreq-v3-reqs ")
+ && line.length() > "dirreq-v3-reqs ".length()) {
+ v3Reqs = line.split(" ")[1];
+ } else if (line.startsWith("dirreq-v3-share ")
+ && v3Reqs != null && !skip) {
+ Map<String, String> obs = new HashMap<String, String>();
+ String[] parts = v3Reqs.split(",");
+ for (String p : parts) {
+ for (String c : this.countries) {
+ if (p.startsWith(c)) {
+ obs.put(c, p.substring(3));
+ }
}
}
- }
- String share = line.substring("dirreq-v3-share ".length(),
- line.length() - 1);
- this.dsfh.addObs(dir, date, obs, share);
- if (!this.lastParsedExtraInfos.containsKey(dir) ||
- statsEnd.compareTo(
- this.lastParsedExtraInfos.get(dir)) > 0) {
- this.lastParsedExtraInfos.put(dir, statsEnd);
- relayDescriptorParseHistoryModified = true;
+ String share = line.substring("dirreq-v3-share ".length(),
+ line.length() - 1);
+ if (this.dsfh != null &&
+ directories.contains(relayIdentifier)) {
+ this.dsfh.addObs(dir, date, obs, share);
+ }
}
}
- }
- }
- }
- /**
- * Returns the URLs of current descriptors that we are missing,
- * including the current consensus and a few extra-info descriptors.
- */
- public Set<String> getMissingDescriptorUrls() {
- Set<String> urls = new HashSet<String>();
- // We might be missing the current consensus for either consensus
- // stats or bridge stats; we remember ourselves which consensus we
- // parsed before (most likely from parsing cached-consensus) and can
- // decide whether we want a more current one
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH");
- format.setTimeZone(TimeZone.getTimeZone("UTC"));
- String currentConsensus = format.format(new Date())
- + ":00:00";
- if (currentConsensus.equals(this.lastParsedConsensus)) {
- urls.add("/tor/status-vote/current/consensus");
- }
- // We might be missing extra-info descriptors for dirreq stats for
- // the directories we care about; we are happy with previous dirreq
- // stats until they are more than 36 hours old (24 hours for the
- // next stats period to end plus 12 hours for publishing a new
- // descriptor)
- format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- format.setTimeZone(TimeZone.getTimeZone("UTC"));
- long now = System.currentTimeMillis();
- for (String directory : this.directories) {
- if (!this.lastParsedExtraInfos.containsKey(directory)) {
- urls.add("/tor/extra/fp/" + directory);
- } else {
- try {
- long statsEnd = format.parse(this.lastParsedExtraInfos.get(
- directory)).getTime();
- if (statsEnd + 36L * 60L * 60L * 1000L < now) {
- urls.add("/tor/extra/fp/" + directory);
- }
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Failed parsing timestamp in "
- + this.relayDescriptorParseHistoryFile.getAbsolutePath()
- + "!", e);
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ String digest = null;
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ if (start >= 0 || sig >= 0 || sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ digest = DigestUtils.shaHex(forDigest);
}
- }
- }
- return urls;
- }
- public void writeFile() {
- if (this.relayDescriptorParseHistoryModified) {
- try {
- this.logger.fine("Writing file "
- + this.relayDescriptorParseHistoryFile.getAbsolutePath()
- + "...");
- this.relayDescriptorParseHistoryFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.relayDescriptorParseHistoryFile));
- bw.write("type,source,published\n");
- if (this.lastParsedConsensus != null) {
- bw.write("consensus,NA," + this.lastParsedConsensus + "\n");
+ if (this.aw != null && digest != null) {
+ this.aw.storeExtraInfoDescriptor(data, digest, published);
}
- for (Map.Entry<String, String> e :
- this.lastParsedExtraInfos.entrySet()) {
- bw.write("extrainfo," + e.getKey() + "," + e.getValue()
- + "\n");
+ if (this.rdd != null && digest != null) {
+ this.rdd.haveParsedExtraInfoDescriptor(publishedTime,
+ relayIdentifier.toLowerCase(), digest);
}
- bw.close();
- this.logger.fine("Finished writing file "
- + this.relayDescriptorParseHistoryFile.getAbsolutePath()
- + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed writing "
- + this.relayDescriptorParseHistoryFile.getAbsolutePath()
- + "!", e);
}
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not parse descriptor. "
+ + "Skipping.", e);
+ } catch (ParseException e) {
+ this.logger.log(Level.WARNING, "Could not parse descriptor. "
+ + "Skipping.", e);
}
}
}
-
--
1.6.5