[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [metrics-db/master] Group classes by kind of processed metrics data.
commit 8746badd1bfd8cb05983159be2336f1cf72cbd44
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Fri Oct 26 12:44:42 2012 -0400
Group classes by kind of processed metrics data.
Also remove unused unit tests.
---
build.xml | 24 +-
src/org/torproject/ernie/db/ArchiveReader.java | 146 ----
src/org/torproject/ernie/db/ArchiveWriter.java | 339 --------
.../ernie/db/BridgeDescriptorParser.java | 46 -
.../ernie/db/BridgePoolAssignmentsProcessor.java | 174 ----
.../torproject/ernie/db/BridgeSnapshotReader.java | 220 -----
.../ernie/db/CachedRelayDescriptorReader.java | 235 -----
src/org/torproject/ernie/db/Configuration.java | 359 --------
.../torproject/ernie/db/ExitListDownloader.java | 100 ---
src/org/torproject/ernie/db/LockFile.java | 52 --
.../torproject/ernie/db/LoggingConfiguration.java | 93 --
src/org/torproject/ernie/db/Main.java | 160 ----
.../ernie/db/RelayDescriptorDownloader.java | 821 ------------------
.../torproject/ernie/db/RelayDescriptorParser.java | 265 ------
src/org/torproject/ernie/db/RsyncDataProvider.java | 217 -----
.../ernie/db/SanitizedBridgesWriter.java | 911 --------------------
src/org/torproject/ernie/db/TorperfDownloader.java | 573 ------------
.../db/bridgedescs/BridgeDescriptorParser.java | 46 +
.../ernie/db/bridgedescs/BridgeSnapshotReader.java | 220 +++++
.../db/bridgedescs/SanitizedBridgesWriter.java | 911 ++++++++++++++++++++
.../BridgePoolAssignmentsProcessor.java | 174 ++++
.../ernie/db/exitlists/ExitListDownloader.java | 100 +++
.../torproject/ernie/db/main/Configuration.java | 359 ++++++++
src/org/torproject/ernie/db/main/LockFile.java | 52 ++
.../ernie/db/main/LoggingConfiguration.java | 93 ++
src/org/torproject/ernie/db/main/Main.java | 172 ++++
.../ernie/db/main/RsyncDataProvider.java | 217 +++++
.../ernie/db/relaydescs/ArchiveReader.java | 146 ++++
.../ernie/db/relaydescs/ArchiveWriter.java | 339 ++++++++
.../db/relaydescs/CachedRelayDescriptorReader.java | 235 +++++
.../db/relaydescs/RelayDescriptorDownloader.java | 821 ++++++++++++++++++
.../ernie/db/relaydescs/RelayDescriptorParser.java | 265 ++++++
.../ernie/db/torperf/TorperfDownloader.java | 573 ++++++++++++
.../org/torproject/ernie/db/ArchiveReaderTest.java | 32 -
.../org/torproject/ernie/db/ArchiveWriterTest.java | 19 -
.../ernie/db/BridgeSnapshotReaderTest.java | 32 -
.../ernie/db/CachedRelayDescriptorReaderTest.java | 31 -
.../ernie/db/SanitizedBridgesWriterTest.java | 38 -
38 files changed, 4724 insertions(+), 4886 deletions(-)
diff --git a/build.xml b/build.xml
index ce3e337..7cba58c 100644
--- a/build.xml
+++ b/build.xml
@@ -1,7 +1,6 @@
<project default="run" name="ERNIE" basedir=".">
<property name="sources" value="src/"/>
<property name="classes" value="classes/"/>
- <property name="tests" value="test"/>
<property name="docs" value="javadoc/"/>
<property name="name" value="ERNIE"/>
<path id="classpath">
@@ -27,7 +26,7 @@
<target name="run" depends="compile">
<java fork="true"
maxmemory="2048m"
- classname="org.torproject.ernie.db.Main">
+ classname="org.torproject.ernie.db.main.Main">
<classpath refid="classpath"/>
</java>
</target>
@@ -38,26 +37,5 @@
<fileset dir="${sources}/" includes="**/*.java" />
</javadoc>
</target>
- <target name="test" depends="compile">
- <javac destdir="${classes}"
- srcdir="${tests}"
- source="1.5"
- target="1.5"
- debug="true"
- deprecation="true"
- optimize="false"
- failonerror="true"
- includeantruntime="false">
- <classpath refid="classpath"/>
- </javac>
- <junit haltonfailure="true" printsummary="off">
- <classpath refid="classpath"/>
- <formatter type="plain" usefile="false"/>
- <batchtest>
- <fileset dir="${classes}"
- includes="**/*Test.class"/>
- </batchtest>
- </junit>
- </target>
</project>
diff --git a/src/org/torproject/ernie/db/ArchiveReader.java b/src/org/torproject/ernie/db/ArchiveReader.java
deleted file mode 100644
index 06abf6c..0000000
--- a/src/org/torproject/ernie/db/ArchiveReader.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
-
-/**
- * Read in all files in a given directory and pass buffered readers of
- * them to the relay descriptor parser.
- */
-public class ArchiveReader {
- public ArchiveReader(RelayDescriptorParser rdp, File archivesDirectory,
- File statsDirectory, boolean keepImportHistory) {
-
- if (rdp == null || archivesDirectory == null ||
- statsDirectory == null) {
- throw new IllegalArgumentException();
- }
-
- int parsedFiles = 0, ignoredFiles = 0;
- Logger logger = Logger.getLogger(ArchiveReader.class.getName());
- SortedSet<String> archivesImportHistory = new TreeSet<String>();
- File archivesImportHistoryFile = new File(statsDirectory,
- "archives-import-history");
- if (keepImportHistory && archivesImportHistoryFile.exists()) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- archivesImportHistoryFile));
- String line = null;
- while ((line = br.readLine()) != null) {
- archivesImportHistory.add(line);
- }
- br.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not read in archives import "
- + "history file. Skipping.");
- }
- }
- if (archivesDirectory.exists()) {
- logger.fine("Importing files in directory " + archivesDirectory
- + "/...");
- Stack<File> filesInInputDir = new Stack<File>();
- filesInInputDir.add(archivesDirectory);
- List<File> problems = new ArrayList<File>();
- while (!filesInInputDir.isEmpty()) {
- File pop = filesInInputDir.pop();
- if (pop.isDirectory()) {
- for (File f : pop.listFiles()) {
- filesInInputDir.add(f);
- }
- } else {
- if (rdp != null) {
- try {
- BufferedInputStream bis = null;
- if (keepImportHistory &&
- archivesImportHistory.contains(pop.getName())) {
- ignoredFiles++;
- continue;
- } else if (pop.getName().endsWith(".tar.bz2")) {
- logger.warning("Cannot parse compressed tarball "
- + pop.getAbsolutePath() + ". Skipping.");
- continue;
- } else if (pop.getName().endsWith(".bz2")) {
- FileInputStream fis = new FileInputStream(pop);
- BZip2CompressorInputStream bcis =
- new BZip2CompressorInputStream(fis);
- bis = new BufferedInputStream(bcis);
- } else {
- FileInputStream fis = new FileInputStream(pop);
- bis = new BufferedInputStream(fis);
- }
- if (keepImportHistory) {
- archivesImportHistory.add(pop.getName());
- }
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- bis.close();
- byte[] allData = baos.toByteArray();
- rdp.parse(allData);
- parsedFiles++;
- } catch (IOException e) {
- problems.add(pop);
- if (problems.size() > 3) {
- break;
- }
- }
- }
- }
- }
- if (problems.isEmpty()) {
- logger.fine("Finished importing files in directory "
- + archivesDirectory + "/.");
- } else {
- StringBuilder sb = new StringBuilder("Failed importing files in "
- + "directory " + archivesDirectory + "/:");
- int printed = 0;
- for (File f : problems) {
- sb.append("\n " + f.getAbsolutePath());
- if (++printed >= 3) {
- sb.append("\n ... more");
- break;
- }
- }
- }
- }
- if (keepImportHistory) {
- try {
- archivesImportHistoryFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- archivesImportHistoryFile));
- for (String line : archivesImportHistory) {
- bw.write(line + "\n");
- }
- bw.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not write archives import "
- + "history file.");
- }
- }
- logger.info("Finished importing relay descriptors from local "
- + "directory:\nParsed " + parsedFiles + ", ignored "
- + ignoredFiles + " files.");
- }
-}
-
diff --git a/src/org/torproject/ernie/db/ArchiveWriter.java b/src/org/torproject/ernie/db/ArchiveWriter.java
deleted file mode 100644
index d1b9499..0000000
--- a/src/org/torproject/ernie/db/ArchiveWriter.java
+++ /dev/null
@@ -1,339 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Hex;
-import org.torproject.descriptor.DescriptorParser;
-import org.torproject.descriptor.DescriptorSourceFactory;
-import org.torproject.descriptor.impl.DescriptorParseException;
-
-public class ArchiveWriter {
- private Logger logger;
- private File outputDirectory;
- private DescriptorParser descriptorParser;
- private int storedConsensuses = 0, storedVotes = 0, storedCerts = 0,
- storedServerDescriptors = 0, storedExtraInfoDescriptors = 0;
-
- public ArchiveWriter(File outputDirectory) {
-
- if (outputDirectory == null) {
- throw new IllegalArgumentException();
- }
-
- this.logger = Logger.getLogger(ArchiveWriter.class.getName());
- this.outputDirectory = outputDirectory;
- this.descriptorParser =
- DescriptorSourceFactory.createDescriptorParser();
- }
-
- private boolean store(byte[] typeAnnotation, byte[] data,
- String filename) {
- try {
- File file = new File(filename);
- if (!file.exists()) {
- this.logger.finer("Storing " + filename);
- if (this.descriptorParser.parseDescriptors(data, filename).size()
- != 1) {
- this.logger.info("Relay descriptor file " + filename
- + " doesn't contain exactly one descriptor. Not storing.");
- return false;
- }
- file.getParentFile().mkdirs();
- BufferedOutputStream bos = new BufferedOutputStream(
- new FileOutputStream(file));
- if (data.length > 0 && data[0] != '@') {
- bos.write(typeAnnotation, 0, typeAnnotation.length);
- }
- bos.write(data, 0, data.length);
- bos.close();
- return true;
- }
- } catch (DescriptorParseException e) {
- this.logger.log(Level.WARNING, "Could not parse relay descriptor "
- + filename + " before storing it to disk. Skipping.", e);
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not store relay descriptor "
- + filename, e);
- }
- return false;
- }
-
- private static final byte[] CONSENSUS_ANNOTATION =
- "@type network-status-consensus-3 1.0\n".getBytes();
- public void storeConsensus(byte[] data, long validAfter) {
- SimpleDateFormat printFormat = new SimpleDateFormat(
- "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
- printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String filename = outputDirectory + "/consensus/"
- + printFormat.format(new Date(validAfter)) + "-consensus";
- if (this.store(CONSENSUS_ANNOTATION, data, filename)) {
- this.storedConsensuses++;
- }
- }
-
- private static final byte[] VOTE_ANNOTATION =
- "@type network-status-vote-3 1.0\n".getBytes();
- public void storeVote(byte[] data, long validAfter,
- String fingerprint, String digest) {
- SimpleDateFormat printFormat = new SimpleDateFormat(
- "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
- printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String filename = outputDirectory + "/vote/"
- + printFormat.format(new Date(validAfter)) + "-vote-"
- + fingerprint + "-" + digest;
- if (this.store(VOTE_ANNOTATION, data, filename)) {
- this.storedVotes++;
- }
- }
-
- private static final byte[] CERTIFICATE_ANNOTATION =
- "@type dir-key-certificate-3 1.0\n".getBytes();
- public void storeCertificate(byte[] data, String fingerprint,
- long published) {
- SimpleDateFormat printFormat = new SimpleDateFormat(
- "yyyy-MM-dd-HH-mm-ss");
- printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String filename = outputDirectory + "/certs/"
- + fingerprint + "-" + printFormat.format(new Date(published));
- if (this.store(CERTIFICATE_ANNOTATION, data, filename)) {
- this.storedCerts++;
- }
- }
-
- private static final byte[] SERVER_DESCRIPTOR_ANNOTATION =
- "@type server-descriptor 1.0\n".getBytes();
- public void storeServerDescriptor(byte[] data, String digest,
- long published) {
- SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
- printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String filename = outputDirectory + "/server-descriptor/"
- + printFormat.format(new Date(published))
- + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
- + digest;
- if (this.store(SERVER_DESCRIPTOR_ANNOTATION, data, filename)) {
- this.storedServerDescriptors++;
- }
- }
-
- private static final byte[] EXTRA_INFO_ANNOTATION =
- "@type extra-info 1.0\n".getBytes();
- public void storeExtraInfoDescriptor(byte[] data,
- String extraInfoDigest, long published) {
- SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
- descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String filename = outputDirectory + "/extra-info/"
- + descriptorFormat.format(new Date(published))
- + extraInfoDigest.substring(0, 1) + "/"
- + extraInfoDigest.substring(1, 2) + "/"
- + extraInfoDigest;
- if (this.store(EXTRA_INFO_ANNOTATION, data, filename)) {
- this.storedExtraInfoDescriptors++;
- }
- }
-
- private StringBuilder intermediateStats = new StringBuilder();
- public void intermediateStats(String event) {
- intermediateStats.append("While " + event + ", we stored "
- + this.storedConsensuses + " consensus(es), " + this.storedVotes
- + " vote(s), " + this.storedCerts + " certificate(s), "
- + this.storedServerDescriptors + " server descriptor(s), and "
- + this.storedExtraInfoDescriptors
- + " extra-info descriptor(s) to disk.\n");
- this.storedConsensuses = 0;
- this.storedVotes = 0;
- this.storedCerts = 0;
- this.storedServerDescriptors = 0;
- this.storedExtraInfoDescriptors = 0;
- }
- /**
- * Dump some statistics on the completeness of descriptors to the logs
- * on level INFO.
- */
- public void dumpStats() {
- StringBuilder sb = new StringBuilder("Finished writing relay "
- + "descriptors to disk.\n");
- sb.append(intermediateStats.toString());
- sb.append("Statistics on the completeness of written relay "
- + "descriptors of the last 3 consensuses (Consensus/Vote, "
- + "valid-after, votes, server descriptors, extra-infos):");
- try {
- SimpleDateFormat validAfterFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- validAfterFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- SimpleDateFormat consensusVoteFormat =
- new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
- consensusVoteFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- SimpleDateFormat descriptorFormat =
- new SimpleDateFormat("yyyy/MM/");
- descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-
- SortedSet<File> consensuses = new TreeSet<File>();
- Stack<File> leftToParse = new Stack<File>();
- leftToParse.add(new File(outputDirectory + "/consensus"));
- while (!leftToParse.isEmpty()) {
- File pop = leftToParse.pop();
- if (pop.isDirectory()) {
- for (File f : pop.listFiles()) {
- leftToParse.add(f);
- }
- } else if (pop.length() > 0) {
- consensuses.add(pop);
- }
- while (consensuses.size() > 3) {
- consensuses.remove(consensuses.first());
- }
- }
- for (File f : consensuses) {
- BufferedReader br = new BufferedReader(new FileReader(f));
- String line = null, validAfterTime = null,
- voteFilenamePrefix = null, dirSource = null;
- int allVotes = 0, foundVotes = 0,
- allServerDescs = 0, foundServerDescs = 0,
- allExtraInfos = 0, foundExtraInfos = 0;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("valid-after ")) {
- validAfterTime = line.substring("valid-after ".length());
- long validAfter = validAfterFormat.parse(
- validAfterTime).getTime();
- voteFilenamePrefix = outputDirectory + "/vote/"
- + consensusVoteFormat.format(new Date(validAfter))
- + "-vote-";
- } else if (line.startsWith("dir-source ")) {
- dirSource = line.split(" ")[2];
- } else if (line.startsWith("vote-digest ")) {
- allVotes++;
- File voteFile = new File(voteFilenamePrefix + dirSource + "-"
- + line.split(" ")[1]);
- if (voteFile.exists()) {
- foundVotes++;
- BufferedReader vbr = new BufferedReader(new FileReader(
- voteFile));
- String line3 = null;
- int voteAllServerDescs = 0, voteFoundServerDescs = 0,
- voteAllExtraInfos = 0, voteFoundExtraInfos = 0;
- while ((line3 = vbr.readLine()) != null) {
- if (line3.startsWith("r ")) {
- voteAllServerDescs++;
- String digest = Hex.encodeHexString(Base64.decodeBase64(
- line3.split(" ")[3] + "=")).toLowerCase();
- long published = validAfterFormat.parse(
- line3.split(" ")[4] + " "
- + line3.split(" ")[5]).getTime();
- String filename = outputDirectory
- + "/server-descriptor/"
- + descriptorFormat.format(new Date(published))
- + digest.substring(0, 1) + "/"
- + digest.substring(1, 2) + "/" + digest;
- if (new File(filename).exists()) {
- BufferedReader sbr = new BufferedReader(new FileReader(
- new File(filename)));
- String line2 = null;
- while ((line2 = sbr.readLine()) != null) {
- if (line2.startsWith("opt extra-info-digest ") ||
- line2.startsWith("extra-info-digest ")) {
- voteAllExtraInfos++;
- String extraInfoDigest = line2.startsWith("opt ") ?
- line2.split(" ")[2].toLowerCase() :
- line2.split(" ")[1].toLowerCase();
- String filename2 =
- outputDirectory.getAbsolutePath()
- + "/extra-info/"
- + descriptorFormat.format(new Date(published))
- + extraInfoDigest.substring(0, 1) + "/"
- + extraInfoDigest.substring(1, 2) + "/"
- + extraInfoDigest;
- if (new File(filename2).exists()) {
- voteFoundExtraInfos++;
- }
- }
- }
- sbr.close();
- voteFoundServerDescs++;
- }
- }
- }
- vbr.close();
- sb.append(String.format("%nV, %s, NA, %d/%d (%.1f%%), "
- + "%d/%d (%.1f%%)", validAfterTime,
- voteFoundServerDescs, voteAllServerDescs,
- 100.0D * (double) voteFoundServerDescs /
- (double) voteAllServerDescs,
- voteFoundExtraInfos, voteAllExtraInfos,
- 100.0D * (double) voteFoundExtraInfos /
- (double) voteAllExtraInfos));
- }
- } else if (line.startsWith("r ")) {
- allServerDescs++;
- String digest = Hex.encodeHexString(Base64.decodeBase64(
- line.split(" ")[3] + "=")).toLowerCase();
- long published = validAfterFormat.parse(
- line.split(" ")[4] + " " + line.split(" ")[5]).getTime();
- String filename = outputDirectory.getAbsolutePath()
- + "/server-descriptor/"
- + descriptorFormat.format(new Date(published))
- + digest.substring(0, 1) + "/"
- + digest.substring(1, 2) + "/" + digest;
- if (new File (filename).exists()) {
- BufferedReader sbr = new BufferedReader(new FileReader(
- new File(filename)));
- String line2 = null;
- while ((line2 = sbr.readLine()) != null) {
- if (line2.startsWith("opt extra-info-digest ") ||
- line2.startsWith("extra-info-digest ")) {
- allExtraInfos++;
- String extraInfoDigest = line2.startsWith("opt ") ?
- line2.split(" ")[2].toLowerCase() :
- line2.split(" ")[1].toLowerCase();
- String filename2 = outputDirectory.getAbsolutePath()
- + "/extra-info/"
- + descriptorFormat.format(new Date(published))
- + extraInfoDigest.substring(0, 1) + "/"
- + extraInfoDigest.substring(1, 2) + "/"
- + extraInfoDigest;
- if (new File (filename2).exists()) {
- foundExtraInfos++;
- }
- }
- }
- sbr.close();
- foundServerDescs++;
- }
- }
- }
- br.close();
- sb.append(String.format("%nC, %s, %d/%d (%.1f%%), "
- + "%d/%d (%.1f%%), %d/%d (%.1f%%)",
- validAfterTime, foundVotes, allVotes,
- 100.0D * (double) foundVotes / (double) allVotes,
- foundServerDescs, allServerDescs,
- 100.0D * (double) foundServerDescs / (double) allServerDescs,
- foundExtraInfos, allExtraInfos,
- 100.0D * (double) foundExtraInfos / (double) allExtraInfos));
- }
- this.logger.info(sb.toString());
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not dump statistics to disk.",
- e);
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not dump statistics to disk.",
- e);
- }
- }
-}
diff --git a/src/org/torproject/ernie/db/BridgeDescriptorParser.java b/src/org/torproject/ernie/db/BridgeDescriptorParser.java
deleted file mode 100644
index 7773525..0000000
--- a/src/org/torproject/ernie/db/BridgeDescriptorParser.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class BridgeDescriptorParser {
- private SanitizedBridgesWriter sbw;
- private Logger logger;
- public BridgeDescriptorParser(SanitizedBridgesWriter sbw) {
- this.sbw = sbw;
- this.logger =
- Logger.getLogger(BridgeDescriptorParser.class.getName());
- }
- public void parse(byte[] allData, String dateTime) {
- try {
- BufferedReader br = new BufferedReader(new StringReader(
- new String(allData, "US-ASCII")));
- String line = br.readLine();
- if (line == null) {
- return;
- } else if (line.startsWith("r ")) {
- if (this.sbw != null) {
- this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime);
- }
- } else if (line.startsWith("router ")) {
- if (this.sbw != null) {
- this.sbw.sanitizeAndStoreServerDescriptor(allData);
- }
- } else if (line.startsWith("extra-info ")) {
- if (this.sbw != null) {
- this.sbw.sanitizeAndStoreExtraInfoDescriptor(allData);
- }
- }
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not parse bridge descriptor.",
- e);
- return;
- }
- }
-}
-
diff --git a/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java
deleted file mode 100644
index d03dcaf..0000000
--- a/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/* Copyright 2011--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.codec.DecoderException;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-
-public class BridgePoolAssignmentsProcessor {
-
- public BridgePoolAssignmentsProcessor(File assignmentsDirectory,
- File sanitizedAssignmentsDirectory) {
-
- Logger logger =
- Logger.getLogger(BridgePoolAssignmentsProcessor.class.getName());
- if (assignmentsDirectory == null ||
- sanitizedAssignmentsDirectory == null) {
- IllegalArgumentException e = new IllegalArgumentException("Neither "
- + "assignmentsDirectory nor sanitizedAssignmentsDirectory may "
- + "be null!");
- throw e;
- }
-
- List<File> assignmentFiles = new ArrayList<File>();
- Stack<File> files = new Stack<File>();
- files.add(assignmentsDirectory);
- while (!files.isEmpty()) {
- File file = files.pop();
- if (file.isDirectory()) {
- files.addAll(Arrays.asList(file.listFiles()));
- } else if (!file.getName().endsWith(".gz")) {
- assignmentFiles.add(file);
- }
- }
-
- SimpleDateFormat assignmentFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- assignmentFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- SimpleDateFormat filenameFormat =
- new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
- filenameFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- for (File assignmentFile : assignmentFiles) {
- logger.info("Processing bridge pool assignment file '"
- + assignmentFile.getAbsolutePath() + "'...");
- try {
- BufferedReader br = null;
- if (assignmentFile.getName().endsWith(".gz")) {
- br = new BufferedReader(new InputStreamReader(
- new GzipCompressorInputStream(new FileInputStream(
- assignmentFile))));
- } else {
- br = new BufferedReader(new FileReader(assignmentFile));
- }
- String line, bridgePoolAssignmentLine = null;
- SortedSet<String> sanitizedAssignments = new TreeSet<String>();
- boolean wroteLastLine = false, skipBefore20120504125947 = true;
- while ((line = br.readLine()) != null || !wroteLastLine) {
- if (line != null && line.startsWith("bridge-pool-assignment ")) {
- String[] parts = line.split(" ");
- if (parts.length != 3) {
- continue;
- }
- /* TODO Take out this temporary hack to ignore all assignments
- * coming from ponticum when byblos was still the official
- * BridgeDB host. */
- if (line.compareTo(
- "bridge-pool-assignment 2012-05-04 12:59:47") >= 0) {
- skipBefore20120504125947 = false;
- }
- }
- if (skipBefore20120504125947) {
- if (line == null) {
- break;
- } else {
- continue;
- }
- }
- if (line == null ||
- line.startsWith("bridge-pool-assignment ")) {
- if (bridgePoolAssignmentLine != null) {
- try {
- long bridgePoolAssignmentTime = assignmentFormat.parse(
- bridgePoolAssignmentLine.substring(
- "bridge-pool-assignment ".length())).getTime();
- File sanitizedAssignmentsFile = new File(
- sanitizedAssignmentsDirectory, filenameFormat.format(
- bridgePoolAssignmentTime));
- if (!sanitizedAssignmentsFile.exists()) {
- sanitizedAssignmentsFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- sanitizedAssignmentsFile));
- bw.write("@type bridge-pool-assignment 1.0\n");
- bw.write(bridgePoolAssignmentLine + "\n");
- for (String assignmentLine : sanitizedAssignments) {
- bw.write(assignmentLine + "\n");
- }
- bw.close();
- }
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not write sanitized "
- + "bridge pool assignment file for line '"
- + bridgePoolAssignmentLine + "' to disk. Skipping "
- + "bridge pool assignment file '"
- + assignmentFile.getAbsolutePath() + "'.", e);
- break;
- } catch (ParseException e) {
- logger.log(Level.WARNING, "Could not write sanitized "
- + "bridge pool assignment file for line '"
- + bridgePoolAssignmentLine + "' to disk. Skipping "
- + "bridge pool assignment file '"
- + assignmentFile.getAbsolutePath() + "'.", e);
- break;
- }
- sanitizedAssignments.clear();
- }
- if (line == null) {
- wroteLastLine = true;
- } else {
- bridgePoolAssignmentLine = line;
- }
- } else {
- String[] parts = line.split(" ");
- if (parts.length < 2 || parts[0].length() < 40) {
- logger.warning("Unrecognized line '" + line
- + "'. Aborting.");
- break;
- }
- String hashedFingerprint = null;
- try {
- hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex(
- line.split(" ")[0].toCharArray())).toLowerCase();
- } catch (DecoderException e) {
- logger.warning("Unable to decode hex fingerprint in line '"
- + line + "'. Aborting.");
- break;
- }
- String assignmentDetails = line.substring(40);
- sanitizedAssignments.add(hashedFingerprint
- + assignmentDetails);
- }
- }
- br.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not read bridge pool assignment "
- + "file '" + assignmentFile.getAbsolutePath()
- + "'. Skipping.", e);
- }
- }
-
- logger.info("Finished processing bridge pool assignment file(s).");
- }
-}
-
diff --git a/src/org/torproject/ernie/db/BridgeSnapshotReader.java b/src/org/torproject/ernie/db/BridgeSnapshotReader.java
deleted file mode 100644
index f21794d..0000000
--- a/src/org/torproject/ernie/db/BridgeSnapshotReader.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-
-/**
- * Reads the half-hourly snapshots of bridge descriptors from Tonga.
- */
-public class BridgeSnapshotReader {
- public BridgeSnapshotReader(BridgeDescriptorParser bdp,
- File bridgeDirectoriesDir, File statsDirectory) {
-
- if (bdp == null || bridgeDirectoriesDir == null ||
- statsDirectory == null) {
- throw new IllegalArgumentException();
- }
-
- Logger logger =
- Logger.getLogger(BridgeSnapshotReader.class.getName());
- SortedSet<String> parsed = new TreeSet<String>();
- File bdDir = bridgeDirectoriesDir;
- File pbdFile = new File(statsDirectory, "parsed-bridge-directories");
- boolean modified = false;
- if (bdDir.exists()) {
- if (pbdFile.exists()) {
- logger.fine("Reading file " + pbdFile.getAbsolutePath() + "...");
- try {
- BufferedReader br = new BufferedReader(new FileReader(pbdFile));
- String line = null;
- while ((line = br.readLine()) != null) {
- parsed.add(line);
- }
- br.close();
- logger.fine("Finished reading file "
- + pbdFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed reading file "
- + pbdFile.getAbsolutePath() + "!", e);
- return;
- }
- }
- logger.fine("Importing files in directory " + bridgeDirectoriesDir
- + "/...");
- Set<String> descriptorImportHistory = new HashSet<String>();
- int parsedFiles = 0, skippedFiles = 0, parsedStatuses = 0,
- parsedServerDescriptors = 0, skippedServerDescriptors = 0,
- parsedExtraInfoDescriptors = 0, skippedExtraInfoDescriptors = 0;
- Stack<File> filesInInputDir = new Stack<File>();
- filesInInputDir.add(bdDir);
- while (!filesInInputDir.isEmpty()) {
- File pop = filesInInputDir.pop();
- if (pop.isDirectory()) {
- for (File f : pop.listFiles()) {
- filesInInputDir.add(f);
- }
- } else if (!parsed.contains(pop.getName())) {
- try {
- FileInputStream in = new FileInputStream(pop);
- if (in.available() > 0) {
- TarArchiveInputStream tais = null;
- if (pop.getName().endsWith(".tar.gz")) {
- GzipCompressorInputStream gcis =
- new GzipCompressorInputStream(in);
- tais = new TarArchiveInputStream(gcis);
- } else if (pop.getName().endsWith(".tar")) {
- tais = new TarArchiveInputStream(in);
- } else {
- continue;
- }
- BufferedInputStream bis = new BufferedInputStream(tais);
- String fn = pop.getName();
- String dateTime = fn.substring(11, 21) + " "
- + fn.substring(22, 24) + ":" + fn.substring(24, 26)
- + ":" + fn.substring(26, 28);
- while ((tais.getNextTarEntry()) != null) {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- byte[] allData = baos.toByteArray();
- if (allData.length == 0) {
- continue;
- }
- String fileDigest = Hex.encodeHexString(DigestUtils.sha(
- allData));
- String ascii = new String(allData, "US-ASCII");
- BufferedReader br3 = new BufferedReader(new StringReader(
- ascii));
- String firstLine = null;
- while ((firstLine = br3.readLine()) != null) {
- if (firstLine.startsWith("@")) {
- continue;
- } else {
- break;
- }
- }
- if (firstLine.startsWith("r ")) {
- bdp.parse(allData, dateTime);
- parsedStatuses++;
- } else if (descriptorImportHistory.contains(fileDigest)) {
- /* Skip server descriptors or extra-info descriptors if
- * we parsed them before. */
- skippedFiles++;
- continue;
- } else {
- int start = -1, sig = -1, end = -1;
- String startToken =
- firstLine.startsWith("router ") ?
- "router " : "extra-info ";
- String sigToken = "\nrouter-signature\n";
- String endToken = "\n-----END SIGNATURE-----\n";
- while (end < ascii.length()) {
- start = ascii.indexOf(startToken, end);
- if (start < 0) {
- break;
- }
- sig = ascii.indexOf(sigToken, start);
- if (sig < 0) {
- break;
- }
- sig += sigToken.length();
- end = ascii.indexOf(endToken, sig);
- if (end < 0) {
- break;
- }
- end += endToken.length();
- byte[] descBytes = new byte[end - start];
- System.arraycopy(allData, start, descBytes, 0,
- end - start);
- String descriptorDigest = Hex.encodeHexString(
- DigestUtils.sha(descBytes));
- if (!descriptorImportHistory.contains(
- descriptorDigest)) {
- bdp.parse(descBytes, dateTime);
- descriptorImportHistory.add(descriptorDigest);
- if (firstLine.startsWith("router ")) {
- parsedServerDescriptors++;
- } else {
- parsedExtraInfoDescriptors++;
- }
- } else {
- if (firstLine.startsWith("router ")) {
- skippedServerDescriptors++;
- } else {
- skippedExtraInfoDescriptors++;
- }
- }
- }
- }
- descriptorImportHistory.add(fileDigest);
- parsedFiles++;
- }
- bis.close();
- }
- in.close();
-
- /* Let's give some memory back, or we'll run out of it. */
- System.gc();
-
- parsed.add(pop.getName());
- modified = true;
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not parse bridge snapshot "
- + pop.getName() + "!", e);
- continue;
- }
- }
- }
- logger.fine("Finished importing files in directory "
- + bridgeDirectoriesDir + "/. In total, we parsed "
- + parsedFiles + " files (skipped " + skippedFiles
- + ") containing " + parsedStatuses + " statuses, "
- + parsedServerDescriptors + " server descriptors (skipped "
- + skippedServerDescriptors + "), and "
- + parsedExtraInfoDescriptors + " extra-info descriptors "
- + "(skipped " + skippedExtraInfoDescriptors + ").");
- if (!parsed.isEmpty() && modified) {
- logger.fine("Writing file " + pbdFile.getAbsolutePath() + "...");
- try {
- pbdFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(pbdFile));
- for (String f : parsed) {
- bw.append(f + "\n");
- }
- bw.close();
- logger.fine("Finished writing file " + pbdFile.getAbsolutePath()
- + ".");
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed writing file "
- + pbdFile.getAbsolutePath() + "!", e);
- }
- }
- }
- }
-}
-
diff --git a/src/org/torproject/ernie/db/CachedRelayDescriptorReader.java b/src/org/torproject/ernie/db/CachedRelayDescriptorReader.java
deleted file mode 100644
index 4da3e44..0000000
--- a/src/org/torproject/ernie/db/CachedRelayDescriptorReader.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.StringReader;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-
-/**
- * Parses all descriptors in local directory cacheddesc/ and sorts them
- * into directory structure in directory-archive/.
- */
-public class CachedRelayDescriptorReader {
- public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
- List<String> inputDirectories, File statsDirectory) {
-
- if (rdp == null || inputDirectories == null ||
- inputDirectories.isEmpty() || statsDirectory == null) {
- throw new IllegalArgumentException();
- }
-
- StringBuilder dumpStats = new StringBuilder("Finished importing "
- + "relay descriptors from local Tor data directories:");
- Logger logger = Logger.getLogger(
- CachedRelayDescriptorReader.class.getName());
-
- /* Read import history containing SHA-1 digests of previously parsed
- * statuses and descriptors, so that we can skip them in this run. */
- Set<String> lastImportHistory = new HashSet<String>(),
- currentImportHistory = new HashSet<String>();
- File importHistoryFile = new File(statsDirectory,
- "cacheddesc-import-history");
- if (importHistoryFile.exists()) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- importHistoryFile));
- String line;
- while ((line = br.readLine()) != null) {
- lastImportHistory.add(line);
- }
- br.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not read import history from "
- + importHistoryFile.getAbsolutePath() + ".", e);
- }
- }
-
- /* Read cached descriptors directories. */
- for (String inputDirectory : inputDirectories) {
- File cachedDescDir = new File(inputDirectory);
- if (!cachedDescDir.exists()) {
- logger.warning("Directory " + cachedDescDir.getAbsolutePath()
- + " does not exist. Skipping.");
- continue;
- }
- logger.fine("Reading " + cachedDescDir.getAbsolutePath()
- + " directory.");
- for (File f : cachedDescDir.listFiles()) {
- try {
- // descriptors may contain non-ASCII chars; read as bytes to
- // determine digests
- BufferedInputStream bis =
- new BufferedInputStream(new FileInputStream(f));
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- bis.close();
- byte[] allData = baos.toByteArray();
- if (f.getName().equals("cached-consensus")) {
- /* Check if directory information is stale. */
- BufferedReader br = new BufferedReader(new StringReader(
- new String(allData, "US-ASCII")));
- String line = null;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("valid-after ")) {
- dumpStats.append("\n" + f.getName() + ": " + line.substring(
- "valid-after ".length()));
- SimpleDateFormat dateTimeFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (dateTimeFormat.parse(line.substring("valid-after ".
- length())).getTime() < System.currentTimeMillis()
- - 6L * 60L * 60L * 1000L) {
- logger.warning("Cached descriptor files in "
- + cachedDescDir.getAbsolutePath() + " are stale. "
- + "The valid-after line in cached-consensus is '"
- + line + "'.");
- dumpStats.append(" (stale!)");
- }
- break;
- }
- }
- br.close();
-
- /* Parse the cached consensus if we haven't parsed it before
- * (but regardless of whether it's stale or not). */
- if (rdp != null) {
- String digest = Hex.encodeHexString(DigestUtils.sha(
- allData));
- if (!lastImportHistory.contains(digest) &&
- !currentImportHistory.contains(digest)) {
- rdp.parse(allData);
- } else {
- dumpStats.append(" (skipped)");
- }
- currentImportHistory.add(digest);
- }
- } else if (f.getName().equals("v3-status-votes")) {
- int parsedNum = 0, skippedNum = 0;
- String ascii = new String(allData, "US-ASCII");
- String startToken = "network-status-version ";
- int end = ascii.length();
- int start = ascii.indexOf(startToken);
- while (start >= 0 && start < end) {
- int next = ascii.indexOf(startToken, start + 1);
- if (next < 0) {
- next = end;
- }
- if (start < next) {
- byte[] rawNetworkStatusBytes = new byte[next - start];
- System.arraycopy(allData, start, rawNetworkStatusBytes, 0,
- next - start);
- if (rdp != null) {
- String digest = Hex.encodeHexString(DigestUtils.sha(
- rawNetworkStatusBytes));
- if (!lastImportHistory.contains(digest) &&
- !currentImportHistory.contains(digest)) {
- rdp.parse(rawNetworkStatusBytes);
- parsedNum++;
- } else {
- skippedNum++;
- }
- currentImportHistory.add(digest);
- }
- }
- start = next;
- }
- dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum
- + ", skipped " + skippedNum + " votes");
- } else if (f.getName().startsWith("cached-descriptors") ||
- f.getName().startsWith("cached-extrainfo")) {
- String ascii = new String(allData, "US-ASCII");
- int start = -1, sig = -1, end = -1;
- String startToken =
- f.getName().startsWith("cached-descriptors") ?
- "router " : "extra-info ";
- String sigToken = "\nrouter-signature\n";
- String endToken = "\n-----END SIGNATURE-----\n";
- int parsedNum = 0, skippedNum = 0;
- while (end < ascii.length()) {
- start = ascii.indexOf(startToken, end);
- if (start < 0) {
- break;
- }
- sig = ascii.indexOf(sigToken, start);
- if (sig < 0) {
- break;
- }
- sig += sigToken.length();
- end = ascii.indexOf(endToken, sig);
- if (end < 0) {
- break;
- }
- end += endToken.length();
- byte[] descBytes = new byte[end - start];
- System.arraycopy(allData, start, descBytes, 0, end - start);
- if (rdp != null) {
- String digest = Hex.encodeHexString(DigestUtils.sha(
- descBytes));
- if (!lastImportHistory.contains(digest) &&
- !currentImportHistory.contains(digest)) {
- rdp.parse(descBytes);
- parsedNum++;
- } else {
- skippedNum++;
- }
- currentImportHistory.add(digest);
- }
- }
- dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum
- + ", skipped " + skippedNum + " "
- + (f.getName().startsWith("cached-descriptors") ?
- "server" : "extra-info") + " descriptors");
- }
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed reading "
- + cachedDescDir.getAbsolutePath() + " directory.", e);
- } catch (ParseException e) {
- logger.log(Level.WARNING, "Failed reading "
- + cachedDescDir.getAbsolutePath() + " directory.", e);
- }
- }
- logger.fine("Finished reading "
- + cachedDescDir.getAbsolutePath() + " directory.");
- }
-
- /* Write import history containing SHA-1 digests to disk. */
- try {
- importHistoryFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- importHistoryFile));
- for (String digest : currentImportHistory) {
- bw.write(digest + "\n");
- }
- bw.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not write import history to "
- + importHistoryFile.getAbsolutePath() + ".", e);
- }
-
- logger.info(dumpStats.toString());
- }
-}
-
diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java
deleted file mode 100644
index e130dab..0000000
--- a/src/org/torproject/ernie/db/Configuration.java
+++ /dev/null
@@ -1,359 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Initialize configuration with hard-coded defaults, overwrite with
- * configuration in config file, if exists, and answer Main.java about our
- * configuration.
- */
-public class Configuration {
- private boolean writeDirectoryArchives = false;
- private String directoryArchivesOutputDirectory = "directory-archive/";
- private boolean importCachedRelayDescriptors = false;
- private List<String> cachedRelayDescriptorsDirectory =
- new ArrayList<String>(Arrays.asList("cacheddesc/".split(",")));
- private boolean importDirectoryArchives = false;
- private String directoryArchivesDirectory = "archives/";
- private boolean keepDirectoryArchiveImportHistory = false;
- private boolean writeSanitizedBridges = false;
- private boolean replaceIPAddressesWithHashes = false;
- private long limitBridgeDescriptorMappings = -1L;
- private String sanitizedBridgesWriteDirectory = "sanitized-bridges/";
- private boolean importBridgeSnapshots = false;
- private String bridgeSnapshotsDirectory = "bridge-directories/";
- private boolean downloadRelayDescriptors = false;
- private List<String> downloadFromDirectoryAuthorities = Arrays.asList((
- "86.59.21.38,76.73.17.194:9030,213.115.239.118:443,"
- + "193.23.244.244,208.83.223.34:443,128.31.0.34:9131,"
- + "194.109.206.212,212.112.245.170").split(","));
- private boolean downloadCurrentConsensus = true;
- private boolean downloadCurrentVotes = true;
- private boolean downloadMissingServerDescriptors = true;
- private boolean downloadMissingExtraInfoDescriptors = true;
- private boolean downloadAllServerDescriptors = false;
- private boolean downloadAllExtraInfoDescriptors = false;
- private boolean compressRelayDescriptorDownloads;
- private boolean downloadExitList = false;
- private boolean processBridgePoolAssignments = false;
- private String assignmentsDirectory = "assignments/";
- private String sanitizedAssignmentsDirectory = "sanitized-assignments/";
- private boolean processTorperfFiles = false;
- private String torperfOutputDirectory = "torperf/";
- private SortedMap<String, String> torperfSources = null;
- private List<String> torperfFiles = null;
- private boolean provideFilesViaRsync = false;
- private String rsyncDirectory = "rsync";
- public Configuration() {
-
- /* Initialize logger. */
- Logger logger = Logger.getLogger(Configuration.class.getName());
-
- /* Read config file, if present. */
- File configFile = new File("config");
- if (!configFile.exists()) {
- logger.warning("Could not find config file. In the default "
- + "configuration, we are not configured to read data from any "
- + "data source or write data to any data sink. You need to "
- + "create a config file (" + configFile.getAbsolutePath()
- + ") and provide at least one data source and one data sink. "
- + "Refer to the manual for more information.");
- return;
- }
- String line = null;
- boolean containsCachedRelayDescriptorsDirectory = false;
- try {
- BufferedReader br = new BufferedReader(new FileReader(configFile));
- while ((line = br.readLine()) != null) {
- if (line.startsWith("#") || line.length() < 1) {
- continue;
- } else if (line.startsWith("WriteDirectoryArchives")) {
- this.writeDirectoryArchives = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DirectoryArchivesOutputDirectory")) {
- this.directoryArchivesOutputDirectory = line.split(" ")[1];
- } else if (line.startsWith("ImportCachedRelayDescriptors")) {
- this.importCachedRelayDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("CachedRelayDescriptorsDirectory")) {
- if (!containsCachedRelayDescriptorsDirectory) {
- this.cachedRelayDescriptorsDirectory.clear();
- containsCachedRelayDescriptorsDirectory = true;
- }
- this.cachedRelayDescriptorsDirectory.add(line.split(" ")[1]);
- } else if (line.startsWith("ImportDirectoryArchives")) {
- this.importDirectoryArchives = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DirectoryArchivesDirectory")) {
- this.directoryArchivesDirectory = line.split(" ")[1];
- } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
- this.keepDirectoryArchiveImportHistory = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("WriteSanitizedBridges")) {
- this.writeSanitizedBridges = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("ReplaceIPAddressesWithHashes")) {
- this.replaceIPAddressesWithHashes = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("LimitBridgeDescriptorMappings")) {
- this.limitBridgeDescriptorMappings = Long.parseLong(
- line.split(" ")[1]);
- } else if (line.startsWith("SanitizedBridgesWriteDirectory")) {
- this.sanitizedBridgesWriteDirectory = line.split(" ")[1];
- } else if (line.startsWith("ImportBridgeSnapshots")) {
- this.importBridgeSnapshots = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("BridgeSnapshotsDirectory")) {
- this.bridgeSnapshotsDirectory = line.split(" ")[1];
- } else if (line.startsWith("DownloadRelayDescriptors")) {
- this.downloadRelayDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadFromDirectoryAuthorities")) {
- this.downloadFromDirectoryAuthorities = new ArrayList<String>();
- for (String dir : line.split(" ")[1].split(",")) {
- // test if IP:port pair has correct format
- if (dir.length() < 1) {
- logger.severe("Configuration file contains directory "
- + "authority IP:port of length 0 in line '" + line
- + "'! Exiting!");
- System.exit(1);
- }
- new URL("http://"; + dir + "/");
- this.downloadFromDirectoryAuthorities.add(dir);
- }
- } else if (line.startsWith("DownloadCurrentConsensus")) {
- this.downloadCurrentConsensus = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadCurrentVotes")) {
- this.downloadCurrentVotes = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadMissingServerDescriptors")) {
- this.downloadMissingServerDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith(
- "DownloadMissingExtraInfoDescriptors")) {
- this.downloadMissingExtraInfoDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadAllServerDescriptors")) {
- this.downloadAllServerDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadAllExtraInfoDescriptors")) {
- this.downloadAllExtraInfoDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("CompressRelayDescriptorDownloads")) {
- this.compressRelayDescriptorDownloads = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadExitList")) {
- this.downloadExitList = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("ProcessBridgePoolAssignments")) {
- this.processBridgePoolAssignments = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("AssignmentsDirectory")) {
- this.assignmentsDirectory = line.split(" ")[1];
- } else if (line.startsWith("SanitizedAssignmentsDirectory")) {
- this.sanitizedAssignmentsDirectory = line.split(" ")[1];
- } else if (line.startsWith("ProcessTorperfFiles")) {
- this.processTorperfFiles = Integer.parseInt(line.split(" ")[1])
- != 0;
- } else if (line.startsWith("TorperfOutputDirectory")) {
- } else if (line.startsWith("TorperfSource")) {
- if (this.torperfSources == null) {
- this.torperfSources = new TreeMap<String, String>();
- }
- String[] parts = line.split(" ");
- String sourceName = parts[1];
- String baseUrl = parts[2];
- this.torperfSources.put(sourceName, baseUrl);
- } else if (line.startsWith("TorperfFiles")) {
- if (this.torperfFiles == null) {
- this.torperfFiles = new ArrayList<String>();
- }
- String[] parts = line.split(" ");
- if (parts.length != 5) {
- logger.severe("Configuration file contains TorperfFiles "
- + "option with wrong number of values in line '" + line
- + "'! Exiting!");
- System.exit(1);
- }
- this.torperfFiles.add(line);
- } else if (line.startsWith("ProvideFilesViaRsync")) {
- this.provideFilesViaRsync = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("RsyncDirectory")) {
- this.rsyncDirectory = line.split(" ")[1];
- } else {
- logger.severe("Configuration file contains unrecognized "
- + "configuration key in line '" + line + "'! Exiting!");
- System.exit(1);
- }
- }
- br.close();
- } catch (ArrayIndexOutOfBoundsException e) {
- logger.severe("Configuration file contains configuration key "
- + "without value in line '" + line + "'. Exiting!");
- System.exit(1);
- } catch (MalformedURLException e) {
- logger.severe("Configuration file contains illegal URL or IP:port "
- + "pair in line '" + line + "'. Exiting!");
- System.exit(1);
- } catch (NumberFormatException e) {
- logger.severe("Configuration file contains illegal value in line '"
- + line + "' with legal values being 0 or 1. Exiting!");
- System.exit(1);
- } catch (IOException e) {
- logger.log(Level.SEVERE, "Unknown problem while reading config "
- + "file! Exiting!", e);
- System.exit(1);
- }
-
- /** Make some checks if configuration is valid. */
- if (!this.importCachedRelayDescriptors &&
- !this.importDirectoryArchives && !this.downloadRelayDescriptors &&
- !this.importBridgeSnapshots &&
- !this.downloadExitList && !this.processBridgePoolAssignments &&
- !this.writeDirectoryArchives && !this.writeSanitizedBridges &&
- !this.processTorperfFiles) {
- logger.warning("We have not been configured to read data from any "
- + "data source or write data to any data sink. You need to "
- + "edit your config file (" + configFile.getAbsolutePath()
- + ") and provide at least one data source and one data sink. "
- + "Refer to the manual for more information.");
- }
- if ((this.importCachedRelayDescriptors ||
- this.importDirectoryArchives || this.downloadRelayDescriptors) &&
- !this.writeDirectoryArchives) {
- logger.warning("We are configured to import/download relay "
- + "descriptors, but we don't have a single data sink to write "
- + "relay descriptors to.");
- }
- if (!(this.importCachedRelayDescriptors ||
- this.importDirectoryArchives || this.downloadRelayDescriptors) &&
- this.writeDirectoryArchives) {
- logger.warning("We are configured to write relay descriptor to at "
- + "least one data sink, but we don't have a single data source "
- + "containing relay descriptors.");
- }
- if (this.importBridgeSnapshots && !this.writeSanitizedBridges) {
- logger.warning("We are configured to import/download bridge "
- + "descriptors, but we don't have a single data sink to write "
- + "bridge descriptors to.");
- }
- if (!this.importBridgeSnapshots && this.writeSanitizedBridges) {
- logger.warning("We are configured to write bridge descriptor to at "
- + "least one data sink, but we don't have a single data source "
- + "containing bridge descriptors.");
- }
- }
- public boolean getWriteDirectoryArchives() {
- return this.writeDirectoryArchives;
- }
- public String getDirectoryArchivesOutputDirectory() {
- return this.directoryArchivesOutputDirectory;
- }
- public boolean getImportCachedRelayDescriptors() {
- return this.importCachedRelayDescriptors;
- }
- public List<String> getCachedRelayDescriptorDirectory() {
- return this.cachedRelayDescriptorsDirectory;
- }
- public boolean getImportDirectoryArchives() {
- return this.importDirectoryArchives;
- }
- public String getDirectoryArchivesDirectory() {
- return this.directoryArchivesDirectory;
- }
- public boolean getKeepDirectoryArchiveImportHistory() {
- return this.keepDirectoryArchiveImportHistory;
- }
- public boolean getWriteSanitizedBridges() {
- return this.writeSanitizedBridges;
- }
- public boolean getReplaceIPAddressesWithHashes() {
- return this.replaceIPAddressesWithHashes;
- }
- public long getLimitBridgeDescriptorMappings() {
- return this.limitBridgeDescriptorMappings;
- }
- public String getSanitizedBridgesWriteDirectory() {
- return this.sanitizedBridgesWriteDirectory;
- }
- public boolean getImportBridgeSnapshots() {
- return this.importBridgeSnapshots;
- }
- public String getBridgeSnapshotsDirectory() {
- return this.bridgeSnapshotsDirectory;
- }
- public boolean getDownloadRelayDescriptors() {
- return this.downloadRelayDescriptors;
- }
- public List<String> getDownloadFromDirectoryAuthorities() {
- return this.downloadFromDirectoryAuthorities;
- }
- public boolean getDownloadCurrentConsensus() {
- return this.downloadCurrentConsensus;
- }
- public boolean getDownloadCurrentVotes() {
- return this.downloadCurrentVotes;
- }
- public boolean getDownloadMissingServerDescriptors() {
- return this.downloadMissingServerDescriptors;
- }
- public boolean getDownloadMissingExtraInfoDescriptors() {
- return this.downloadMissingExtraInfoDescriptors;
- }
- public boolean getDownloadAllServerDescriptors() {
- return this.downloadAllServerDescriptors;
- }
- public boolean getDownloadAllExtraInfoDescriptors() {
- return this.downloadAllExtraInfoDescriptors;
- }
- public boolean getCompressRelayDescriptorDownloads() {
- return this.compressRelayDescriptorDownloads;
- }
- public boolean getDownloadExitList() {
- return this.downloadExitList;
- }
- public boolean getProcessBridgePoolAssignments() {
- return processBridgePoolAssignments;
- }
- public String getAssignmentsDirectory() {
- return assignmentsDirectory;
- }
- public String getSanitizedAssignmentsDirectory() {
- return sanitizedAssignmentsDirectory;
- }
- public boolean getProcessTorperfFiles() {
- return this.processTorperfFiles;
- }
- public String getTorperfOutputDirectory() {
- return this.torperfOutputDirectory;
- }
- public SortedMap<String, String> getTorperfSources() {
- return this.torperfSources;
- }
- public List<String> getTorperfFiles() {
- return this.torperfFiles;
- }
- public boolean getProvideFilesViaRsync() {
- return this.provideFilesViaRsync;
- }
- public String getRsyncDirectory() {
- return this.rsyncDirectory;
- }
-}
-
diff --git a/src/org/torproject/ernie/db/ExitListDownloader.java b/src/org/torproject/ernie/db/ExitListDownloader.java
deleted file mode 100644
index 01a554f..0000000
--- a/src/org/torproject/ernie/db/ExitListDownloader.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class ExitListDownloader {
- public ExitListDownloader() {
- Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
- try {
- logger.fine("Downloading exit list...");
- String exitAddressesUrl =
- "http://exitlist.torproject.org/exit-addresses";;
- URL u = new URL(exitAddressesUrl);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- int response = huc.getResponseCode();
- if (response != 200) {
- logger.warning("Could not download exit list. Response code " +
- response);
- return;
- }
- BufferedInputStream in = new BufferedInputStream(
- huc.getInputStream());
- SimpleDateFormat printFormat =
- new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
- printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- Date downloadedDate = new Date();
- File exitListFile = new File("exitlist/" + printFormat.format(
- downloadedDate));
- exitListFile.getParentFile().mkdirs();
- SimpleDateFormat dateTimeFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- exitListFile));
- bw.write("@type tordnsel 1.0\n");
- bw.write("Downloaded " + dateTimeFormat.format(downloadedDate)
- + "\n");
- int len;
- byte[] data = new byte[1024];
- while ((len = in.read(data, 0, 1024)) >= 0) {
- bw.write(new String(data, 0, len));
- }
- in.close();
- bw.close();
- logger.fine("Finished downloading exit list.");
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed downloading exit list", e);
- return;
- }
-
- /* Write stats. */
- StringBuilder dumpStats = new StringBuilder("Finished downloading "
- + "exit list.\nLast three exit lists are:");
- Stack<File> filesInInputDir = new Stack<File>();
- filesInInputDir.add(new File("exitlist"));
- SortedSet<File> lastThreeExitLists = new TreeSet<File>();
- while (!filesInInputDir.isEmpty()) {
- File pop = filesInInputDir.pop();
- if (pop.isDirectory()) {
- SortedSet<File> lastThreeElements = new TreeSet<File>();
- for (File f : pop.listFiles()) {
- lastThreeElements.add(f);
- }
- while (lastThreeElements.size() > 3) {
- lastThreeElements.remove(lastThreeElements.first());
- }
- for (File f : lastThreeElements) {
- filesInInputDir.add(f);
- }
- } else {
- lastThreeExitLists.add(pop);
- while (lastThreeExitLists.size() > 3) {
- lastThreeExitLists.remove(lastThreeExitLists.first());
- }
- }
- }
- for (File f : lastThreeExitLists) {
- dumpStats.append("\n" + f.getName());
- }
- logger.info(dumpStats.toString());
- }
-}
-
diff --git a/src/org/torproject/ernie/db/LockFile.java b/src/org/torproject/ernie/db/LockFile.java
deleted file mode 100644
index 3255620..0000000
--- a/src/org/torproject/ernie/db/LockFile.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.logging.Logger;
-
-public class LockFile {
-
- private File lockFile;
- private Logger logger;
-
- public LockFile() {
- this.lockFile = new File("lock");
- this.logger = Logger.getLogger(LockFile.class.getName());
- }
-
- public boolean acquireLock() {
- this.logger.fine("Trying to acquire lock...");
- try {
- if (this.lockFile.exists()) {
- BufferedReader br = new BufferedReader(new FileReader("lock"));
- long runStarted = Long.parseLong(br.readLine());
- br.close();
- if (System.currentTimeMillis() - runStarted < 55L * 60L * 1000L) {
- return false;
- }
- }
- BufferedWriter bw = new BufferedWriter(new FileWriter("lock"));
- bw.append("" + System.currentTimeMillis() + "\n");
- bw.close();
- this.logger.fine("Acquired lock.");
- return true;
- } catch (IOException e) {
- this.logger.warning("Caught exception while trying to acquire "
- + "lock!");
- return false;
- }
- }
-
- public void releaseLock() {
- this.logger.fine("Releasing lock...");
- this.lockFile.delete();
- this.logger.fine("Released lock.");
- }
-}
-
diff --git a/src/org/torproject/ernie/db/LoggingConfiguration.java b/src/org/torproject/ernie/db/LoggingConfiguration.java
deleted file mode 100644
index b83ef53..0000000
--- a/src/org/torproject/ernie/db/LoggingConfiguration.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedWriter;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.TimeZone;
-import java.util.logging.ConsoleHandler;
-import java.util.logging.FileHandler;
-import java.util.logging.Formatter;
-import java.util.logging.Handler;
-import java.util.logging.Level;
-import java.util.logging.LogRecord;
-import java.util.logging.Logger;
-/**
- * Initialize logging configuration.
- *
- * Log levels used by ERNIE:
- *
- * - SEVERE: An event made it impossible to continue program execution.
- * - WARNING: A potential problem occurred that requires the operator to
- * look after the otherwise unattended setup
- * - INFO: Messages on INFO level are meant to help the operator in making
- * sure that operation works as expected.
- * - FINE: Debug messages that are used to identify problems and which are
- * turned on by default.
- * - FINER: More detailed debug messages to investigate problems in more
- * detail. Not turned on by default. Increase log file limit when using
- * FINER.
- * - FINEST: Most detailed debug messages. Not used.
- */
-public class LoggingConfiguration {
- public LoggingConfiguration() {
-
- /* Remove default console handler. */
- for (Handler h : Logger.getLogger("").getHandlers()) {
- Logger.getLogger("").removeHandler(h);
- }
-
- /* Disable logging of internal Sun classes. */
- Logger.getLogger("sun").setLevel(Level.OFF);
-
- /* Set minimum log level we care about from INFO to FINER. */
- Logger.getLogger("").setLevel(Level.FINER);
-
- /* Create log handler that writes messages on WARNING or higher to the
- * console. */
- final SimpleDateFormat dateTimeFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- Formatter cf = new Formatter() {
- public String format(LogRecord record) {
- return dateTimeFormat.format(new Date(record.getMillis())) + " "
- + record.getMessage() + "\n";
- }
- };
- Handler ch = new ConsoleHandler();
- ch.setFormatter(cf);
- ch.setLevel(Level.WARNING);
- Logger.getLogger("").addHandler(ch);
-
- /* Initialize own logger for this class. */
- Logger logger = Logger.getLogger(
- LoggingConfiguration.class.getName());
-
- /* Create log handler that writes all messages on FINE or higher to a
- * local file. */
- Formatter ff = new Formatter() {
- public String format(LogRecord record) {
- return dateTimeFormat.format(new Date(record.getMillis())) + " "
- + record.getLevel() + " " + record.getSourceClassName() + " "
- + record.getSourceMethodName() + " " + record.getMessage()
- + (record.getThrown() != null ? " " + record.getThrown() : "")
- + "\n";
- }
- };
- try {
- FileHandler fh = new FileHandler("log", 5000000, 5, true);
- fh.setFormatter(ff);
- fh.setLevel(Level.FINE);
- Logger.getLogger("").addHandler(fh);
- } catch (SecurityException e) {
- logger.log(Level.WARNING, "No permission to create log file. "
- + "Logging to file is disabled.", e);
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not write to log file. Logging to "
- + "file is disabled.", e);
- }
- }
-}
diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java
deleted file mode 100644
index 04cc868..0000000
--- a/src/org/torproject/ernie/db/Main.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.File;
-import java.util.List;
-import java.util.logging.Logger;
-
-/**
- * Coordinate downloading and parsing of descriptors and extraction of
- * statistically relevant data for later processing with R.
- */
-public class Main {
- public static void main(String[] args) {
-
- /* Initialize logging configuration. */
- new LoggingConfiguration();
-
- Logger logger = Logger.getLogger(Main.class.getName());
- logger.info("Starting ERNIE.");
-
- // Initialize configuration
- Configuration config = new Configuration();
-
- // Use lock file to avoid overlapping runs
- LockFile lf = new LockFile();
- if (!lf.acquireLock()) {
- logger.severe("Warning: ERNIE is already running or has not exited "
- + "cleanly! Exiting!");
- System.exit(1);
- }
-
- // Define stats directory for temporary files
- File statsDirectory = new File("stats");
-
- // Prepare writing relay descriptor archive to disk
- ArchiveWriter aw = config.getWriteDirectoryArchives() ?
- new ArchiveWriter(
- new File(config.getDirectoryArchivesOutputDirectory())) : null;
-
- // Prepare relay descriptor parser (only if we are writing stats or
- // directory archives to disk)
- RelayDescriptorParser rdp = aw != null ?
- new RelayDescriptorParser(aw) : null;
-
- // Import/download relay descriptors from the various sources
- if (rdp != null) {
- RelayDescriptorDownloader rdd = null;
- if (config.getDownloadRelayDescriptors()) {
- List<String> dirSources =
- config.getDownloadFromDirectoryAuthorities();
- rdd = new RelayDescriptorDownloader(rdp, dirSources,
- config.getDownloadCurrentConsensus(),
- config.getDownloadCurrentVotes(),
- config.getDownloadMissingServerDescriptors(),
- config.getDownloadMissingExtraInfoDescriptors(),
- config.getDownloadAllServerDescriptors(),
- config.getDownloadAllExtraInfoDescriptors(),
- config.getCompressRelayDescriptorDownloads());
- rdp.setRelayDescriptorDownloader(rdd);
- }
- if (config.getImportCachedRelayDescriptors()) {
- new CachedRelayDescriptorReader(rdp,
- config.getCachedRelayDescriptorDirectory(), statsDirectory);
- if (aw != null) {
- aw.intermediateStats("importing relay descriptors from local "
- + "Tor data directories");
- }
- }
- if (config.getImportDirectoryArchives()) {
- new ArchiveReader(rdp,
- new File(config.getDirectoryArchivesDirectory()),
- statsDirectory,
- config.getKeepDirectoryArchiveImportHistory());
- if (aw != null) {
- aw.intermediateStats("importing relay descriptors from local "
- + "directory");
- }
- }
- if (rdd != null) {
- rdd.downloadDescriptors();
- rdd.writeFile();
- rdd = null;
- if (aw != null) {
- aw.intermediateStats("downloading relay descriptors from the "
- + "directory authorities");
- }
- }
- }
-
- // Write output to disk that only depends on relay descriptors
- if (aw != null) {
- aw.dumpStats();
- aw = null;
- }
-
- // Prepare sanitized bridge descriptor writer
- SanitizedBridgesWriter sbw = config.getWriteSanitizedBridges() ?
- new SanitizedBridgesWriter(
- new File(config.getSanitizedBridgesWriteDirectory()),
- statsDirectory, config.getReplaceIPAddressesWithHashes(),
- config.getLimitBridgeDescriptorMappings()) : null;
-
- // Prepare bridge descriptor parser
- BridgeDescriptorParser bdp = config.getWriteSanitizedBridges()
- ? new BridgeDescriptorParser(sbw) : null;
-
- // Import bridge descriptors
- if (bdp != null && config.getImportBridgeSnapshots()) {
- new BridgeSnapshotReader(bdp,
- new File(config.getBridgeSnapshotsDirectory()),
- statsDirectory);
- }
-
- // Finish writing sanitized bridge descriptors to disk
- if (sbw != null) {
- sbw.finishWriting();
- sbw = null;
- }
-
- // Download exit list and store it to disk
- if (config.getDownloadExitList()) {
- new ExitListDownloader();
- }
-
- // Process bridge pool assignments
- if (config.getProcessBridgePoolAssignments()) {
- new BridgePoolAssignmentsProcessor(
- new File(config.getAssignmentsDirectory()),
- new File(config.getSanitizedAssignmentsDirectory()));
- }
-
- // Process Torperf files
- if (config.getProcessTorperfFiles()) {
- new TorperfDownloader(new File(config.getTorperfOutputDirectory()),
- config.getTorperfSources(), config.getTorperfFiles());
- }
-
- // Copy recently published files to a local directory that can then
- // be served via rsync.
- if (config.getProvideFilesViaRsync()) {
- new RsyncDataProvider(
- !config.getWriteDirectoryArchives() ? null :
- new File(config.getDirectoryArchivesOutputDirectory()),
- !config.getWriteSanitizedBridges() ? null :
- new File(config.getSanitizedBridgesWriteDirectory()),
- !config.getProcessBridgePoolAssignments() ? null :
- new File(config.getSanitizedAssignmentsDirectory()),
- config.getDownloadExitList(),
- !config.getProcessTorperfFiles() ? null :
- new File(config.getTorperfOutputDirectory()),
- new File(config.getRsyncDirectory()));
- }
-
- // Remove lock file
- lf.releaseLock();
-
- logger.info("Terminating ERNIE.");
- }
-}
diff --git a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java b/src/org/torproject/ernie/db/RelayDescriptorDownloader.java
deleted file mode 100644
index f7e9468..0000000
--- a/src/org/torproject/ernie/db/RelayDescriptorDownloader.java
+++ /dev/null
@@ -1,821 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.zip.InflaterInputStream;
-
-/**
- * Downloads relay descriptors from the directory authorities via HTTP.
- * Keeps a list of missing descriptors that gets updated by parse results
- * from <code>RelayDescriptorParser</code> and downloads all missing
- * descriptors that have been published in the last 24 hours. Also
- * downloads all server and extra-info descriptors known to a directory
- * authority at most once a day.
- */
-public class RelayDescriptorDownloader {
-
- /**
- * Text file containing the descriptors that we are missing and that we
- * want to download. Lines are formatted as:
- *
- * - "consensus,<validafter>,<parsed>",
- * - "vote,<validafter>,<fingerprint>,<parsed>",
- * - "server,<published>,<relayid>,<descid>,<parsed>", or
- * - "extra,<published>,<relayid>,<descid>,<parsed>".
- */
- private File missingDescriptorsFile;
-
- /**
- * Relay descriptors that we are missing and that we want to download
- * either in this execution or write to disk and try next time. Map keys
- * contain comma-separated values as in the missing descriptors files
- * without the "parsed" column. Map values contain the "parsed" column.
- */
- private SortedMap<String, String> missingDescriptors;
-
- /**
- * Text file containing the IP addresses (and Dir ports if not 80) of
- * directory authorities and when we last downloaded all server and
- * extra-info descriptors from them, so that we can avoid downloading
- * them too often.
- */
- private File lastDownloadedAllDescriptorsFile;
-
- /**
- * Map of directory authorities and when we last downloaded all server
- * and extra-info descriptors from them. Map keys are IP addresses (and
- * Dir ports if not 80), map values are timestamps.
- */
- private Map<String, String> lastDownloadedAllDescriptors;
-
- /**
- * <code>RelayDescriptorParser</code> that we will hand over the
- * downloaded descriptors for parsing.
- */
- private RelayDescriptorParser rdp;
-
- /**
- * Directory authorities that we will try to download missing
- * descriptors from.
- */
- private List<String> authorities;
-
- /**
- * Should we try to download the current consensus if we don't have it?
- */
- private boolean downloadCurrentConsensus;
-
- /**
- * Should we try to download current votes if we don't have them?
- */
- private boolean downloadCurrentVotes;
-
- /**
- * Should we try to download missing server descriptors that have been
- * published within the past 24 hours?
- */
- private boolean downloadMissingServerDescriptors;
-
- /**
- * Should we try to download missing extra-info descriptors that have
- * been published within the past 24 hours?
- */
- private boolean downloadMissingExtraInfos;
-
- /**
- * Should we try to download all server descriptors from the authorities
- * once every 24 hours?
- */
- private boolean downloadAllServerDescriptors;
-
- /**
- * Should we try to download all extra-info descriptors from the
- * authorities once every 24 hours?
- */
- private boolean downloadAllExtraInfos;
-
- /**
- * Should we download zlib-compressed versions of descriptors by adding
- * ".z" to URLs?
- */
- private boolean downloadCompressed;
-
- /**
- * valid-after time that we expect the current consensus and votes to
- * have, formatted "yyyy-MM-dd HH:mm:ss". We only expect to find
- * consensuses and votes with this valid-after time on the directory
- * authorities. This time is initialized as the beginning of the current
- * hour.
- */
- private String currentValidAfter;
-
- /**
- * Cut-off time for missing server and extra-info descriptors, formatted
- * "yyyy-MM-dd HH:mm:ss". This time is initialized as the current system
- * time minus 24 hours.
- */
- private String descriptorCutOff;
-
- /**
- * Cut-off time for downloading all server and extra-info descriptors
- * from the directory authorities, formatted "yyyy-MM-dd HH:mm:ss". This
- * time is initialized as the current system time minus 23:30 hours.
- */
- private String downloadAllDescriptorsCutOff;
-
- /**
- * Directory authorities that we plan to download all server and
- * extra-info descriptors from in this execution.
- */
- private Set<String> downloadAllDescriptorsFromAuthorities;
-
- /**
- * Current timestamp that is written to the missing list for descriptors
- * that we parsed in this execution and for authorities that we
- * downloaded all server and extra-info descriptors from.
- */
- private String currentTimestamp;
-
- /**
- * Logger for this class.
- */
- private Logger logger;
-
- /**
- * Number of descriptors requested by directory authority to be included
- * in logs.
- */
- private Map<String, Integer> requestsByAuthority;
-
- /**
- * Counters for descriptors that we had on the missing list at the
- * beginning of the execution, that we added to the missing list,
- * that we requested, and that we successfully downloaded in this
- * execution.
- */
- private int oldMissingConsensuses = 0, oldMissingVotes = 0,
- oldMissingServerDescriptors = 0, oldMissingExtraInfoDescriptors = 0,
- newMissingConsensuses = 0, newMissingVotes = 0,
- newMissingServerDescriptors = 0, newMissingExtraInfoDescriptors = 0,
- requestedConsensuses = 0, requestedVotes = 0,
- requestedMissingServerDescriptors = 0,
- requestedAllServerDescriptors = 0,
- requestedMissingExtraInfoDescriptors = 0,
- requestedAllExtraInfoDescriptors = 0, downloadedConsensuses = 0,
- downloadedVotes = 0, downloadedMissingServerDescriptors = 0,
- downloadedAllServerDescriptors = 0,
- downloadedMissingExtraInfoDescriptors = 0,
- downloadedAllExtraInfoDescriptors = 0;
-
- /**
- * Initializes this class, including reading in missing descriptors from
- * <code>stats/missing-relay-descriptors</code> and the times when we
- * last downloaded all server and extra-info descriptors from
- * <code>stats/last-downloaded-all-descriptors</code>.
- */
- public RelayDescriptorDownloader(RelayDescriptorParser rdp,
- List<String> authorities, boolean downloadCurrentConsensus,
- boolean downloadCurrentVotes,
- boolean downloadMissingServerDescriptors,
- boolean downloadMissingExtraInfos,
- boolean downloadAllServerDescriptors, boolean downloadAllExtraInfos,
- boolean downloadCompressed) {
-
- /* Memorize argument values. */
- this.rdp = rdp;
- this.authorities = new ArrayList<String>(authorities);
- this.downloadCurrentConsensus = downloadCurrentConsensus;
- this.downloadCurrentVotes = downloadCurrentVotes;
- this.downloadMissingServerDescriptors =
- downloadMissingServerDescriptors;
- this.downloadMissingExtraInfos = downloadMissingExtraInfos;
- this.downloadAllServerDescriptors = downloadAllServerDescriptors;
- this.downloadAllExtraInfos = downloadAllExtraInfos;
- this.downloadCompressed = downloadCompressed;
-
- /* Shuffle list of authorities for better load balancing over time. */
- Collections.shuffle(this.authorities);
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(
- RelayDescriptorDownloader.class.getName());
-
- /* Prepare cut-off times and timestamp for the missing descriptors
- * list and the list of authorities to download all server and
- * extra-info descriptors from. */
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- format.setTimeZone(TimeZone.getTimeZone("UTC"));
- long now = System.currentTimeMillis();
- this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) *
- (60L * 60L * 1000L));
- this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L);
- this.currentTimestamp = format.format(now);
- this.downloadAllDescriptorsCutOff = format.format(now
- - 23L * 60L * 60L * 1000L - 30L * 60L * 1000L);
-
- /* Read list of missing descriptors from disk and memorize those that
- * we are interested in and that are likely to be found on the
- * directory authorities. */
- this.missingDescriptors = new TreeMap<String, String>();
- this.missingDescriptorsFile = new File(
- "stats/missing-relay-descriptors");
- if (this.missingDescriptorsFile.exists()) {
- try {
- this.logger.fine("Reading file "
- + this.missingDescriptorsFile.getAbsolutePath() + "...");
- BufferedReader br = new BufferedReader(new FileReader(
- this.missingDescriptorsFile));
- String line;
- while ((line = br.readLine()) != null) {
- if (line.split(",").length > 2) {
- String published = line.split(",")[1];
- if (((line.startsWith("consensus,") ||
- line.startsWith("vote,")) &&
- this.currentValidAfter.equals(published)) ||
- ((line.startsWith("server,") ||
- line.startsWith("extra,")) &&
- this.descriptorCutOff.compareTo(published) < 0)) {
- if (!line.endsWith("NA")) {
- /* Not missing. */
- } else if (line.startsWith("consensus,")) {
- oldMissingConsensuses++;
- } else if (line.startsWith("vote,")) {
- oldMissingVotes++;
- } else if (line.startsWith("server,")) {
- oldMissingServerDescriptors++;
- } else if (line.startsWith("extra,")) {
- oldMissingExtraInfoDescriptors++;
- }
- int separateAt = line.lastIndexOf(",");
- this.missingDescriptors.put(line.substring(0,
- separateAt), line.substring(separateAt + 1));
- }
- } else {
- this.logger.fine("Invalid line '" + line + "' in "
- + this.missingDescriptorsFile.getAbsolutePath()
- + ". Ignoring.");
- }
- }
- br.close();
- this.logger.fine("Finished reading file "
- + this.missingDescriptorsFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to read file "
- + this.missingDescriptorsFile.getAbsolutePath()
- + "! This means that we might forget to dowload relay "
- + "descriptors we are missing.", e);
- }
- }
-
- /* Read list of directory authorities and when we last downloaded all
- * server and extra-info descriptors from them. */
- this.lastDownloadedAllDescriptors = new HashMap<String, String>();
- this.lastDownloadedAllDescriptorsFile = new File(
- "stats/last-downloaded-all-descriptors");
- if (this.lastDownloadedAllDescriptorsFile.exists()) {
- try {
- this.logger.fine("Reading file "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
- + "...");
- BufferedReader br = new BufferedReader(new FileReader(
- this.lastDownloadedAllDescriptorsFile));
- String line;
- while ((line = br.readLine()) != null) {
- if (line.split(",").length != 2) {
- this.logger.fine("Invalid line '" + line + "' in "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
- + ". Ignoring.");
- } else {
- String[] parts = line.split(",");
- String authority = parts[0];
- String lastDownloaded = parts[1];
- this.lastDownloadedAllDescriptors.put(authority,
- lastDownloaded);
- }
- }
- br.close();
- this.logger.fine("Finished reading file "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
- + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to read file "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
- + "! This means that we might download all server and "
- + "extra-info descriptors more often than we should.", e);
- }
- }
-
- /* Make a list of at most two directory authorities that we want to
- * download all server and extra-info descriptors from. */
- this.downloadAllDescriptorsFromAuthorities = new HashSet<String>();
- for (String authority : this.authorities) {
- if (!this.lastDownloadedAllDescriptors.containsKey(authority) ||
- this.lastDownloadedAllDescriptors.get(authority).compareTo(
- this.downloadAllDescriptorsCutOff) < 0) {
- this.downloadAllDescriptorsFromAuthorities.add(authority);
- }
- if (this.downloadAllDescriptorsFromAuthorities.size() >= 2) {
- break;
- }
- }
-
- /* Prepare statistics on this execution. */
- this.requestsByAuthority = new HashMap<String, Integer>();
- for (String authority : this.authorities) {
- this.requestsByAuthority.put(authority, 0);
- }
- }
-
- /**
- * We have parsed a consensus. Take this consensus off the missing list
- * and add the votes created by the given <code>authorities</code> and
- * the <code>serverDescriptors</code> which are in the format
- * "<published>,<relayid>,<descid>" to that list.
- */
- public void haveParsedConsensus(String validAfter,
- Set<String> authorities, Set<String> serverDescriptors) {
-
- /* Mark consensus as parsed. */
- if (this.currentValidAfter.equals(validAfter)) {
- String consensusKey = "consensus," + validAfter;
- this.missingDescriptors.put(consensusKey, this.currentTimestamp);
-
- /* Add votes to missing list. */
- for (String authority : authorities) {
- String voteKey = "vote," + validAfter + "," + authority;
- if (!this.missingDescriptors.containsKey(voteKey)) {
- this.missingDescriptors.put(voteKey, "NA");
- this.newMissingVotes++;
- }
- }
- }
-
- /* Add server descriptors to missing list. */
- for (String serverDescriptor : serverDescriptors) {
- String published = serverDescriptor.split(",")[0];
- if (this.descriptorCutOff.compareTo(published) < 0) {
- String serverDescriptorKey = "server," + serverDescriptor;
- if (!this.missingDescriptors.containsKey(
- serverDescriptorKey)) {
- this.missingDescriptors.put(serverDescriptorKey, "NA");
- this.newMissingServerDescriptors++;
- }
- }
- }
- }
-
- /**
- * We have parsed a vote. Take this vote off the missing list and add
- * the <code>serverDescriptors</code> which are in the format
- * "<published>,<relayid>,<descid>" to that list.
- */
- public void haveParsedVote(String validAfter, String fingerprint,
- Set<String> serverDescriptors) {
-
- /* Mark vote as parsed. */
- if (this.currentValidAfter.equals(validAfter)) {
- String voteKey = "vote," + validAfter + "," + fingerprint;
- this.missingDescriptors.put(voteKey, this.currentTimestamp);
- }
-
- /* Add server descriptors to missing list. */
- for (String serverDescriptor : serverDescriptors) {
- String published = serverDescriptor.split(",")[0];
- if (this.descriptorCutOff.compareTo(published) < 0) {
- String serverDescriptorKey = "server," + serverDescriptor;
- if (!this.missingDescriptors.containsKey(
- serverDescriptorKey)) {
- this.missingDescriptors.put(serverDescriptorKey, "NA");
- this.newMissingServerDescriptors++;
- }
- }
- }
- }
-
- /**
- * We have parsed a server descriptor. Take this server descriptor off
- * the missing list and put the extra-info descriptor digest on that
- * list.
- */
- public void haveParsedServerDescriptor(String published,
- String relayIdentity, String serverDescriptorDigest,
- String extraInfoDigest) {
-
- /* Mark server descriptor as parsed. */
- if (this.descriptorCutOff.compareTo(published) < 0) {
- String serverDescriptorKey = "server," + published + ","
- + relayIdentity + "," + serverDescriptorDigest;
- this.missingDescriptors.put(serverDescriptorKey,
- this.currentTimestamp);
-
- /* Add extra-info descriptor to missing list. */
- if (extraInfoDigest != null) {
- String extraInfoKey = "extra," + published + ","
- + relayIdentity + "," + extraInfoDigest;
- if (!this.missingDescriptors.containsKey(extraInfoKey)) {
- this.missingDescriptors.put(extraInfoKey, "NA");
- this.newMissingExtraInfoDescriptors++;
- }
- }
- }
- }
-
- /**
- * We have parsed an extra-info descriptor. Take it off the missing
- * list.
- */
- public void haveParsedExtraInfoDescriptor(String published,
- String relayIdentity, String extraInfoDigest) {
- if (this.descriptorCutOff.compareTo(published) < 0) {
- String extraInfoKey = "extra," + published + ","
- + relayIdentity + "," + extraInfoDigest;
- this.missingDescriptors.put(extraInfoKey, this.currentTimestamp);
- }
- }
-
- /**
- * Downloads missing descriptors that we think might still be available
- * on the directory authorities as well as all server and extra-info
- * descriptors once per day.
- */
- public void downloadDescriptors() {
-
- /* Put the current consensus on the missing list, unless we already
- * have it. */
- String consensusKey = "consensus," + this.currentValidAfter;
- if (!this.missingDescriptors.containsKey(consensusKey)) {
- this.missingDescriptors.put(consensusKey, "NA");
- this.newMissingConsensuses++;
- }
-
- /* Download descriptors from authorities which are in random order, so
- * that we distribute the load somewhat fairly over time. */
- for (String authority : authorities) {
-
- /* Make all requests to an authority in a single try block. If
- * something goes wrong with this authority, we give up on all
- * downloads and continue with the next authority. */
- /* TODO Some authorities provide very little bandwidth and could
- * slow down the entire download process. Ponder adding a timeout of
- * 3 or 5 minutes per authority to avoid getting in the way of the
- * next execution. */
- try {
-
- /* Start with downloading the current consensus, unless we already
- * have it. */
- if (downloadCurrentConsensus) {
- if (this.missingDescriptors.containsKey(consensusKey) &&
- this.missingDescriptors.get(consensusKey).equals("NA")) {
- this.requestedConsensuses++;
- this.downloadedConsensuses +=
- this.downloadResourceFromAuthority(authority,
- "/tor/status-vote/current/consensus");
- }
- }
-
- /* Next, try to download current votes that we're missing. */
- if (downloadCurrentVotes) {
- String voteKeyPrefix = "vote," + this.currentValidAfter;
- SortedSet<String> fingerprints = new TreeSet<String>();
- for (Map.Entry<String, String> e :
- this.missingDescriptors.entrySet()) {
- if (e.getValue().equals("NA") &&
- e.getKey().startsWith(voteKeyPrefix)) {
- String fingerprint = e.getKey().split(",")[2];
- fingerprints.add(fingerprint);
- }
- }
- for (String fingerprint : fingerprints) {
- this.requestedVotes++;
- this.downloadedVotes +=
- this.downloadResourceFromAuthority(authority,
- "/tor/status-vote/current/" + fingerprint);
- }
- }
-
- /* Download either all server and extra-info descriptors or only
- * those that we're missing. Start with server descriptors, then
- * request extra-info descriptors. */
- List<String> types = new ArrayList<String>(Arrays.asList(
- "server,extra".split(",")));
- for (String type : types) {
-
- /* Download all server or extra-info descriptors from this
- * authority if we haven't done so for 24 hours and if we're
- * configured to do so. */
- if (this.downloadAllDescriptorsFromAuthorities.contains(
- authority) && ((type.equals("server") &&
- this.downloadAllServerDescriptors) ||
- (type.equals("extra") && this.downloadAllExtraInfos))) {
- int downloadedAllDescriptors =
- this.downloadResourceFromAuthority(authority, "/tor/"
- + type + "/all");
- if (type.equals("server")) {
- this.requestedAllServerDescriptors++;
- this.downloadedAllServerDescriptors +=
- downloadedAllDescriptors;
- } else {
- this.requestedAllExtraInfoDescriptors++;
- this.downloadedAllExtraInfoDescriptors +=
- downloadedAllDescriptors;
- }
-
- /* Download missing server or extra-info descriptors if we're
- * configured to do so. */
- } else if ((type.equals("server") &&
- this.downloadMissingServerDescriptors) ||
- (type.equals("extra") && this.downloadMissingExtraInfos)) {
-
- /* Go through the list of missing descriptors of this type
- * and combine the descriptor identifiers to a URL of up to
- * 96 descriptors that we can download at once. */
- SortedSet<String> descriptorIdentifiers =
- new TreeSet<String>();
- for (Map.Entry<String, String> e :
- this.missingDescriptors.entrySet()) {
- if (e.getValue().equals("NA") &&
- e.getKey().startsWith(type + ",") &&
- this.descriptorCutOff.compareTo(
- e.getKey().split(",")[1]) < 0) {
- String descriptorIdentifier = e.getKey().split(",")[3];
- descriptorIdentifiers.add(descriptorIdentifier);
- }
- }
- StringBuilder combinedResource = null;
- int descriptorsInCombinedResource = 0,
- requestedDescriptors = 0, downloadedDescriptors = 0;
- for (String descriptorIdentifier : descriptorIdentifiers) {
- if (descriptorsInCombinedResource >= 96) {
- requestedDescriptors += descriptorsInCombinedResource;
- downloadedDescriptors +=
- this.downloadResourceFromAuthority(authority,
- combinedResource.toString());
- combinedResource = null;
- descriptorsInCombinedResource = 0;
- }
- if (descriptorsInCombinedResource == 0) {
- combinedResource = new StringBuilder("/tor/" + type
- + "/d/" + descriptorIdentifier);
- } else {
- combinedResource.append("+" + descriptorIdentifier);
- }
- descriptorsInCombinedResource++;
- }
- if (descriptorsInCombinedResource > 0) {
- requestedDescriptors += descriptorsInCombinedResource;
- downloadedDescriptors +=
- this.downloadResourceFromAuthority(authority,
- combinedResource.toString());
- }
- if (type.equals("server")) {
- this.requestedMissingServerDescriptors +=
- requestedDescriptors;
- this.downloadedMissingServerDescriptors +=
- downloadedDescriptors;
- } else {
- this.requestedMissingExtraInfoDescriptors +=
- requestedDescriptors;
- this.downloadedMissingExtraInfoDescriptors +=
- downloadedDescriptors;
- }
- }
- }
-
- /* If a download failed, stop requesting descriptors from this
- * authority and move on to the next. */
- } catch (IOException e) {
- logger.log(Level.FINE, "Failed downloading from " + authority
- + "!", e);
- }
- }
- }
-
- /**
- * Attempts to download one or more descriptors identified by a resource
- * string from a directory authority and passes the returned
- * descriptor(s) to the <code>RelayDescriptorParser</code> upon success.
- * Returns the number of descriptors contained in the reply. Throws an
- * <code>IOException</code> if something goes wrong while downloading.
- */
- private int downloadResourceFromAuthority(String authority,
- String resource) throws IOException {
- byte[] allData = null;
- this.requestsByAuthority.put(authority,
- this.requestsByAuthority.get(authority) + 1);
- /* TODO Disable compressed downloads for extra-info descriptors,
- * because zlib decompression doesn't work correctly. Figure out why
- * this is and fix it. */
- String fullUrl = "http://"; + authority + resource
- + (this.downloadCompressed && !resource.startsWith("/tor/extra/")
- ? ".z" : "");
- URL u = new URL(fullUrl);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- int response = huc.getResponseCode();
- if (response == 200) {
- BufferedInputStream in = this.downloadCompressed &&
- !resource.startsWith("/tor/extra/")
- ? new BufferedInputStream(new InflaterInputStream(
- huc.getInputStream()))
- : new BufferedInputStream(huc.getInputStream());
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = in.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- in.close();
- allData = baos.toByteArray();
- }
- logger.fine("Downloaded " + fullUrl + " -> " + response + " ("
- + (allData == null ? 0 : allData.length) + " bytes)");
- int receivedDescriptors = 0;
- if (allData != null) {
- if (resource.startsWith("/tor/status-vote/current/")) {
- this.rdp.parse(allData);
- receivedDescriptors = 1;
- } else if (resource.startsWith("/tor/server/") ||
- resource.startsWith("/tor/extra/")) {
- if (resource.equals("/tor/server/all")) {
- this.lastDownloadedAllDescriptors.put(authority,
- this.currentTimestamp);
- }
- String ascii = null;
- try {
- ascii = new String(allData, "US-ASCII");
- } catch (UnsupportedEncodingException e) {
- /* No way that US-ASCII is not supported. */
- }
- int start = -1, sig = -1, end = -1;
- String startToken = resource.startsWith("/tor/server/") ?
- "router " : "extra-info ";
- String sigToken = "\nrouter-signature\n";
- String endToken = "\n-----END SIGNATURE-----\n";
- while (end < ascii.length()) {
- start = ascii.indexOf(startToken, end);
- if (start < 0) {
- break;
- }
- sig = ascii.indexOf(sigToken, start);
- if (sig < 0) {
- break;
- }
- sig += sigToken.length();
- end = ascii.indexOf(endToken, sig);
- if (end < 0) {
- break;
- }
- end += endToken.length();
- byte[] descBytes = new byte[end - start];
- System.arraycopy(allData, start, descBytes, 0, end - start);
- this.rdp.parse(descBytes);
- receivedDescriptors++;
- }
- }
- }
- return receivedDescriptors;
- }
-
- /**
- * Writes status files to disk and logs statistics about downloading
- * relay descriptors in this execution.
- */
- public void writeFile() {
-
- /* Write missing descriptors file to disk. */
- int missingConsensuses = 0, missingVotes = 0,
- missingServerDescriptors = 0, missingExtraInfoDescriptors = 0;
- try {
- this.logger.fine("Writing file "
- + this.missingDescriptorsFile.getAbsolutePath() + "...");
- this.missingDescriptorsFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.missingDescriptorsFile));
- for (Map.Entry<String, String> e :
- this.missingDescriptors.entrySet()) {
- String key = e.getKey(), value = e.getValue();
- if (!value.equals("NA")) {
- /* Not missing. */
- } else if (key.startsWith("consensus,")) {
- missingConsensuses++;
- } else if (key.startsWith("vote,")) {
- missingVotes++;
- } else if (key.startsWith("server,")) {
- missingServerDescriptors++;
- } else if (key.startsWith("extra,")) {
- missingExtraInfoDescriptors++;
- }
- bw.write(key + "," + value + "\n");
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.missingDescriptorsFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed writing "
- + this.missingDescriptorsFile.getAbsolutePath() + "!", e);
- }
-
- /* Write text file containing the directory authorities and when we
- * last downloaded all server and extra-info descriptors from them to
- * disk. */
- try {
- this.logger.fine("Writing file "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
- + "...");
- this.lastDownloadedAllDescriptorsFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.lastDownloadedAllDescriptorsFile));
- for (Map.Entry<String, String> e :
- this.lastDownloadedAllDescriptors.entrySet()) {
- String authority = e.getKey();
- String lastDownloaded = e.getValue();
- bw.write(authority + "," + lastDownloaded + "\n");
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
- + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed writing "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + "!",
- e);
- }
-
- /* Log statistics about this execution. */
- this.logger.info("Finished downloading relay descriptors from the "
- + "directory authorities.");
- this.logger.info("At the beginning of this execution, we were "
- + "missing " + oldMissingConsensuses + " consensus(es), "
- + oldMissingVotes + " vote(s), " + oldMissingServerDescriptors
- + " server descriptor(s), and " + oldMissingExtraInfoDescriptors
- + " extra-info descriptor(s).");
- this.logger.info("During this execution, we added "
- + this.newMissingConsensuses + " consensus(es), "
- + this.newMissingVotes + " vote(s), "
- + this.newMissingServerDescriptors + " server descriptor(s), and "
- + this.newMissingExtraInfoDescriptors + " extra-info "
- + "descriptor(s) to the missing list, some of which we also "
- + "requested and removed from the list again.");
- this.logger.info("We requested " + this.requestedConsensuses
- + " consensus(es), " + this.requestedVotes + " vote(s), "
- + this.requestedMissingServerDescriptors + " missing server "
- + "descriptor(s), " + this.requestedAllServerDescriptors
- + " times all server descriptors, "
- + this.requestedMissingExtraInfoDescriptors + " missing "
- + "extra-info descriptor(s), and "
- + this.requestedAllExtraInfoDescriptors + " times all extra-info "
- + "descriptors from the directory authorities.");
- StringBuilder sb = new StringBuilder();
- for (String authority : this.authorities) {
- sb.append(" " + authority + "="
- + this.requestsByAuthority.get(authority));
- }
- this.logger.info("We sent these numbers of requests to the directory "
- + "authorities:" + sb.toString());
- this.logger.info("We successfully downloaded "
- + this.downloadedConsensuses + " consensus(es), "
- + this.downloadedVotes + " vote(s), "
- + this.downloadedMissingServerDescriptors + " missing server "
- + "descriptor(s), " + this.downloadedAllServerDescriptors
- + " server descriptor(s) when downloading all descriptors, "
- + this.downloadedMissingExtraInfoDescriptors + " missing "
- + "extra-info descriptor(s) and "
- + this.downloadedAllExtraInfoDescriptors + " extra-info "
- + "descriptor(s) when downloading all descriptors.");
- this.logger.info("At the end of this execution, we are missing "
- + missingConsensuses + " consensus(es), " + missingVotes
- + " vote(s), " + missingServerDescriptors + " server "
- + "descriptor(s), and " + missingExtraInfoDescriptors
- + " extra-info descriptor(s), some of which we may try in the next "
- + "execution.");
- }
-}
-
diff --git a/src/org/torproject/ernie/db/RelayDescriptorParser.java b/src/org/torproject/ernie/db/RelayDescriptorParser.java
deleted file mode 100644
index be54656..0000000
--- a/src/org/torproject/ernie/db/RelayDescriptorParser.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.StringReader;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.SortedSet;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-
-/**
- * Parses relay descriptors including network status consensuses and
- * votes, server and extra-info descriptors, and passes the results to the
- * stats handlers, to the archive writer, or to the relay descriptor
- * downloader.
- */
-public class RelayDescriptorParser {
-
- /**
- * File writer that writes descriptor contents to files in a
- * directory-archive directory structure.
- */
- private ArchiveWriter aw;
-
- /**
- * Missing descriptor downloader that uses the parse results to learn
- * which descriptors we are missing and want to download.
- */
- private RelayDescriptorDownloader rdd;
-
- /**
- * Logger for this class.
- */
- private Logger logger;
-
- private SimpleDateFormat dateTimeFormat;
-
- /**
- * Initializes this class.
- */
- public RelayDescriptorParser(ArchiveWriter aw) {
- this.aw = aw;
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
-
- this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- }
-
- public void setRelayDescriptorDownloader(
- RelayDescriptorDownloader rdd) {
- this.rdd = rdd;
- }
-
- public void parse(byte[] data) {
- try {
- /* Convert descriptor to ASCII for parsing. This means we'll lose
- * the non-ASCII chars, but we don't care about them for parsing
- * anyway. */
- BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
- String line;
- do {
- line = br.readLine();
- } while (line != null && line.startsWith("@"));
- if (line == null) {
- this.logger.fine("We were given an empty descriptor for "
- + "parsing. Ignoring.");
- return;
- }
- SimpleDateFormat parseFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (line.equals("network-status-version 3")) {
- // TODO when parsing the current consensus, check the fresh-until
- // time to see when we switch from hourly to half-hourly
- // consensuses
- boolean isConsensus = true;
- String validAfterTime = null, fingerprint = null,
- dirSource = null;
- long validAfter = -1L, dirKeyPublished = -1L;
- SortedSet<String> dirSources = new TreeSet<String>();
- SortedSet<String> serverDescriptors = new TreeSet<String>();
- SortedSet<String> hashedRelayIdentities = new TreeSet<String>();
- StringBuilder certificateStringBuilder = null;
- String certificateString = null;
- while ((line = br.readLine()) != null) {
- if (certificateStringBuilder != null) {
- if (line.startsWith("r ")) {
- certificateString = certificateStringBuilder.toString();
- certificateStringBuilder = null;
- } else {
- certificateStringBuilder.append(line + "\n");
- }
- }
- if (line.equals("vote-status vote")) {
- isConsensus = false;
- } else if (line.startsWith("valid-after ")) {
- validAfterTime = line.substring("valid-after ".length());
- validAfter = parseFormat.parse(validAfterTime).getTime();
- } else if (line.startsWith("dir-source ")) {
- dirSource = line.split(" ")[2];
- } else if (line.startsWith("vote-digest ")) {
- dirSources.add(dirSource);
- } else if (line.startsWith("dir-key-certificate-version ")) {
- certificateStringBuilder = new StringBuilder();
- certificateStringBuilder.append(line + "\n");
- } else if (line.startsWith("fingerprint ")) {
- fingerprint = line.split(" ")[1];
- } else if (line.startsWith("dir-key-published ")) {
- String dirKeyPublishedTime = line.substring(
- "dir-key-published ".length());
- dirKeyPublished = parseFormat.parse(dirKeyPublishedTime).
- getTime();
- } else if (line.startsWith("r ")) {
- String[] parts = line.split(" ");
- if (parts.length < 9) {
- this.logger.log(Level.WARNING, "Could not parse r line '"
- + line + "' in descriptor. Skipping.");
- break;
- }
- String publishedTime = parts[4] + " " + parts[5];
- String relayIdentity = Hex.encodeHexString(
- Base64.decodeBase64(parts[2] + "=")).
- toLowerCase();
- String serverDesc = Hex.encodeHexString(Base64.decodeBase64(
- parts[3] + "=")).toLowerCase();
- serverDescriptors.add(publishedTime + "," + relayIdentity
- + "," + serverDesc);
- hashedRelayIdentities.add(DigestUtils.shaHex(
- Base64.decodeBase64(parts[2] + "=")).
- toUpperCase());
- }
- }
- if (isConsensus) {
- if (this.rdd != null) {
- this.rdd.haveParsedConsensus(validAfterTime, dirSources,
- serverDescriptors);
- }
- if (this.aw != null) {
- this.aw.storeConsensus(data, validAfter);
- }
- } else {
- if (this.aw != null || this.rdd != null) {
- String ascii = new String(data, "US-ASCII");
- String startToken = "network-status-version ";
- String sigToken = "directory-signature ";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken);
- if (start >= 0 && sig >= 0 && sig > start) {
- sig += sigToken.length();
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- String digest = DigestUtils.shaHex(forDigest).toUpperCase();
- if (this.aw != null) {
- this.aw.storeVote(data, validAfter, dirSource, digest);
- }
- if (this.rdd != null) {
- this.rdd.haveParsedVote(validAfterTime, fingerprint,
- serverDescriptors);
- }
- }
- if (certificateString != null) {
- if (this.aw != null) {
- this.aw.storeCertificate(certificateString.getBytes(),
- dirSource, dirKeyPublished);
- }
- }
- }
- }
- } else if (line.startsWith("router ")) {
- String publishedTime = null, extraInfoDigest = null,
- relayIdentifier = null;
- long published = -1L;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("published ")) {
- publishedTime = line.substring("published ".length());
- published = parseFormat.parse(publishedTime).getTime();
- } else if (line.startsWith("opt fingerprint") ||
- line.startsWith("fingerprint")) {
- relayIdentifier = line.substring(line.startsWith("opt ") ?
- "opt fingerprint".length() : "fingerprint".length()).
- replaceAll(" ", "").toLowerCase();
- } else if (line.startsWith("opt extra-info-digest ") ||
- line.startsWith("extra-info-digest ")) {
- extraInfoDigest = line.startsWith("opt ") ?
- line.split(" ")[2].toLowerCase() :
- line.split(" ")[1].toLowerCase();
- }
- }
- String ascii = new String(data, "US-ASCII");
- String startToken = "router ";
- String sigToken = "\nrouter-signature\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- String digest = null;
- if (start >= 0 || sig >= 0 || sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- digest = DigestUtils.shaHex(forDigest);
- }
- if (this.aw != null && digest != null) {
- this.aw.storeServerDescriptor(data, digest, published);
- }
- if (this.rdd != null && digest != null) {
- this.rdd.haveParsedServerDescriptor(publishedTime,
- relayIdentifier, digest, extraInfoDigest);
- }
- } else if (line.startsWith("extra-info ")) {
- String publishedTime = null, relayIdentifier = line.split(" ")[2];
- long published = -1L;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("published ")) {
- publishedTime = line.substring("published ".length());
- published = parseFormat.parse(publishedTime).getTime();
- }
- }
- String ascii = new String(data, "US-ASCII");
- String startToken = "extra-info ";
- String sigToken = "\nrouter-signature\n";
- String digest = null;
- int start = ascii.indexOf(startToken);
- if (start > 0) {
- /* Do not confuse "extra-info " in "@type extra-info 1.0" with
- * "extra-info 0000...". TODO This is a hack that should be
- * solved by using metrics-lib some day. */
- start = ascii.indexOf("\n" + startToken);
- if (start > 0) {
- start++;
- }
- }
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- if (start >= 0 || sig >= 0 || sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- digest = DigestUtils.shaHex(forDigest);
- }
- if (this.aw != null && digest != null) {
- this.aw.storeExtraInfoDescriptor(data, digest, published);
- }
- if (this.rdd != null && digest != null) {
- this.rdd.haveParsedExtraInfoDescriptor(publishedTime,
- relayIdentifier.toLowerCase(), digest);
- }
- }
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not parse descriptor. "
- + "Skipping.", e);
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not parse descriptor. "
- + "Skipping.", e);
- }
- }
-}
-
diff --git a/src/org/torproject/ernie/db/RsyncDataProvider.java b/src/org/torproject/ernie/db/RsyncDataProvider.java
deleted file mode 100644
index 2f9632e..0000000
--- a/src/org/torproject/ernie/db/RsyncDataProvider.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/* Copyright 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedInputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.Stack;
-import java.util.logging.Logger;
-
-/**
- * Copy files published in the last 3 days to a local directory that can
- * then be served via rsync.
- */
-public class RsyncDataProvider {
- public RsyncDataProvider(File directoryArchivesOutputDirectory,
- File sanitizedBridgesWriteDirectory,
- File sanitizedAssignmentsDirectory,
- boolean downloadExitList,
- File torperfOutputDirectory, File rsyncDirectory) {
-
- /* Initialize logger. */
- Logger logger = Logger.getLogger(RsyncDataProvider.class.getName());
-
- /* Determine the cut-off time for files in rsync/. */
- long cutOffMillis = System.currentTimeMillis()
- - 3L * 24L * 60L * 60L * 1000L;
-
- /* Create rsync/ directory if it doesn't exist. */
- if (!rsyncDirectory.exists()) {
- rsyncDirectory.mkdirs();
- }
-
- /* Make a list of all files in the rsync/ directory to delete those
- * that we didn't copy in this run. */
- Set<String> fileNamesInRsync = new HashSet<String>();
- Stack<File> files = new Stack<File>();
- files.add(rsyncDirectory);
- while (!files.isEmpty()) {
- File pop = files.pop();
- if (pop.isDirectory()) {
- files.addAll(Arrays.asList(pop.listFiles()));
- } else {
- fileNamesInRsync.add(pop.getName());
- }
- }
- logger.info("Found " + fileNamesInRsync.size() + " files in "
- + rsyncDirectory.getAbsolutePath() + " that we're either "
- + "overwriting or deleting in this execution.");
-
- /* Copy relay descriptors from the last 3 days. */
- if (directoryArchivesOutputDirectory != null) {
- files.add(directoryArchivesOutputDirectory);
- while (!files.isEmpty()) {
- File pop = files.pop();
- if (pop.isDirectory()) {
- files.addAll(Arrays.asList(pop.listFiles()));
- } else if (pop.lastModified() >= cutOffMillis) {
- String fileName = pop.getName();
- if (pop.getAbsolutePath().contains("/consensus/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "relay-descriptors/consensuses/" + fileName));
- } else if (pop.getAbsolutePath().contains("/vote/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "relay-descriptors/votes/" + fileName));
- } else if (pop.getAbsolutePath().contains(
- "/server-descriptor/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "relay-descriptors/server-descriptors/" + fileName));
- } else if (pop.getAbsolutePath().contains("/extra-info/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "relay-descriptors/extra-infos/" + fileName));
- } else {
- continue;
- }
- fileNamesInRsync.remove(pop.getName());
- }
- }
- }
- logger.info("After copying relay descriptors, there are still "
- + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
-
- /* Copy sanitized bridge descriptors from the last 3 days. */
- if (sanitizedBridgesWriteDirectory != null) {
- files.add(sanitizedBridgesWriteDirectory);
- while (!files.isEmpty()) {
- File pop = files.pop();
- if (pop.isDirectory()) {
- files.addAll(Arrays.asList(pop.listFiles()));
- } else if (pop.lastModified() >= cutOffMillis) {
- String fileName = pop.getName();
- if (pop.getAbsolutePath().contains("/statuses/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "bridge-descriptors/statuses/" + fileName));
- } else if (pop.getAbsolutePath().contains(
- "/server-descriptors/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "bridge-descriptors/server-descriptors/" + fileName));
- } else if (pop.getAbsolutePath().contains("/extra-infos/")) {
- this.copyFile(pop, new File(rsyncDirectory,
- "bridge-descriptors/extra-infos/" + fileName));
- } else {
- continue;
- }
- fileNamesInRsync.remove(pop.getName());
- }
- }
- }
- logger.info("After copying sanitized bridge descriptors, there are "
- + "still " + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
-
- /* Copy sanitized bridge pool assignments from the last 3 days. */
- if (sanitizedAssignmentsDirectory != null) {
- files.add(sanitizedAssignmentsDirectory);
- while (!files.isEmpty()) {
- File pop = files.pop();
- if (pop.isDirectory()) {
- files.addAll(Arrays.asList(pop.listFiles()));
- } else if (pop.lastModified() >= cutOffMillis) {
- String fileName = pop.getName();
- this.copyFile(pop, new File(rsyncDirectory,
- "bridge-pool-assignments/" + fileName));
- fileNamesInRsync.remove(pop.getName());
- }
- }
- }
- logger.info("After copying sanitized bridge pool assignments, there "
- + "are still " + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
-
- /* Copy exit lists from the last 3 days. */
- if (downloadExitList) {
- files.add(new File("exitlist"));
- while (!files.isEmpty()) {
- File pop = files.pop();
- if (pop.isDirectory()) {
- files.addAll(Arrays.asList(pop.listFiles()));
- } else if (pop.lastModified() >= cutOffMillis) {
- String fileName = pop.getName();
- this.copyFile(pop, new File(rsyncDirectory,
- "exit-lists/" + fileName));
- fileNamesInRsync.remove(pop.getName());
- }
- }
- }
- logger.info("After copying exit lists, there are still "
- + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
-
- /* Copy Torperf files. */
- if (torperfOutputDirectory != null) {
- files.add(torperfOutputDirectory);
- while (!files.isEmpty()) {
- File pop = files.pop();
- if (pop.isDirectory()) {
- files.addAll(Arrays.asList(pop.listFiles()));
- } else if (pop.getName().endsWith(".tpf") &&
- pop.lastModified() >= cutOffMillis) {
- String fileName = pop.getName();
- this.copyFile(pop, new File(rsyncDirectory,
- "torperf/" + fileName));
- fileNamesInRsync.remove(pop.getName());
- }
- }
- }
- logger.info("After copying Torperf files, there are still "
- + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
-
- /* Delete all files that we didn't (over-)write in this run. */
- files.add(rsyncDirectory);
- while (!files.isEmpty()) {
- File pop = files.pop();
- if (pop.isDirectory()) {
- files.addAll(Arrays.asList(pop.listFiles()));
- } else if (fileNamesInRsync.contains(pop.getName())) {
- fileNamesInRsync.remove(pop.getName());
- pop.delete();
- }
- }
- logger.info("After deleting files that we didn't overwrite in this "
- + "run, there are " + fileNamesInRsync.size() + " files left in "
- + rsyncDirectory.getAbsolutePath() + ".");
- }
-
- private void copyFile(File from, File to) {
- if (from.exists() && to.exists() &&
- from.lastModified() == to.lastModified() &&
- from.length() == to.length()) {
- return;
- }
- try {
- to.getParentFile().mkdirs();
- FileInputStream fis = new FileInputStream(from);
- BufferedInputStream bis = new BufferedInputStream(fis);
- FileOutputStream fos = new FileOutputStream(to);
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- fos.write(data, 0, len);
- }
- bis.close();
- fos.close();
- to.setLastModified(from.lastModified());
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-}
-
diff --git a/src/org/torproject/ernie/db/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
deleted file mode 100644
index afafe11..0000000
--- a/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
+++ /dev/null
@@ -1,911 +0,0 @@
-/* Copyright 2010--2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.*;
-import java.security.*;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.logging.*;
-
-import org.apache.commons.codec.DecoderException;
-import org.apache.commons.codec.digest.*;
-import org.apache.commons.codec.binary.*;
-
-/**
- * Sanitizes bridge descriptors, i.e., removes all possibly sensitive
- * information from them, and writes them to a local directory structure.
- * During the sanitizing process, all information about the bridge
- * identity or IP address are removed or replaced. The goal is to keep the
- * sanitized bridge descriptors useful for statistical analysis while not
- * making it easier for an adversary to enumerate bridges.
- *
- * There are three types of bridge descriptors: bridge network statuses
- * (lists of all bridges at a given time), server descriptors (published
- * by the bridge to advertise their capabilities), and extra-info
- * descriptors (published by the bridge, mainly for statistical analysis).
- */
-public class SanitizedBridgesWriter {
-
- /**
- * Logger for this class.
- */
- private Logger logger;
-
- /**
- * Output directory for writing sanitized bridge descriptors.
- */
- private File sanitizedBridgesDirectory;
-
- private boolean replaceIPAddressesWithHashes;
-
- private boolean persistenceProblemWithSecrets;
-
- private SortedMap<String, byte[]> secretsForHashingIPAddresses;
-
- private String bridgeSanitizingCutOffTimestamp;
-
- private boolean haveWarnedAboutInterval;
-
- private File bridgeIpSecretsFile;
-
- private SecureRandom secureRandom;
-
- /**
- * Initializes this class.
- */
- public SanitizedBridgesWriter(File sanitizedBridgesDirectory,
- File statsDirectory, boolean replaceIPAddressesWithHashes,
- long limitBridgeSanitizingInterval) {
-
- if (sanitizedBridgesDirectory == null || statsDirectory == null) {
- throw new IllegalArgumentException();
- }
-
- /* Memorize argument values. */
- this.sanitizedBridgesDirectory = sanitizedBridgesDirectory;
- this.replaceIPAddressesWithHashes = replaceIPAddressesWithHashes;
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(
- SanitizedBridgesWriter.class.getName());
-
- /* Initialize secure random number generator if we need it. */
- if (this.replaceIPAddressesWithHashes) {
- try {
- this.secureRandom = SecureRandom.getInstance("SHA1PRNG", "SUN");
- } catch (GeneralSecurityException e) {
- this.logger.log(Level.WARNING, "Could not initialize secure "
- + "random number generator! Not calculating any IP address "
- + "hashes in this execution!", e);
- this.persistenceProblemWithSecrets = true;
- }
- }
-
- /* Read hex-encoded secrets for replacing IP addresses with hashes
- * from disk. */
- this.secretsForHashingIPAddresses = new TreeMap<String, byte[]>();
- this.bridgeIpSecretsFile = new File(statsDirectory,
- "bridge-ip-secrets");
- if (this.bridgeIpSecretsFile.exists()) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- this.bridgeIpSecretsFile));
- String line;
- while ((line = br.readLine()) != null) {
- String[] parts = line.split(",");
- if ((line.length() != ("yyyy-MM,".length() + 31 * 2) &&
- line.length() != ("yyyy-MM,".length() + 50 * 2)) ||
- parts.length != 2) {
- this.logger.warning("Invalid line in bridge-ip-secrets file "
- + "starting with '" + line.substring(0, 7) + "'! "
- + "Not calculating any IP address hashes in this "
- + "execution!");
- this.persistenceProblemWithSecrets = true;
- break;
- }
- String month = parts[0];
- byte[] secret = Hex.decodeHex(parts[1].toCharArray());
- this.secretsForHashingIPAddresses.put(month, secret);
- }
- br.close();
- if (!this.persistenceProblemWithSecrets) {
- this.logger.fine("Read "
- + this.secretsForHashingIPAddresses.size() + " secrets for "
- + "hashing bridge IP addresses.");
- }
- } catch (DecoderException e) {
- this.logger.log(Level.WARNING, "Failed to decode hex string in "
- + this.bridgeIpSecretsFile + "! Not calculating any IP "
- + "address hashes in this execution!", e);
- this.persistenceProblemWithSecrets = true;
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to read "
- + this.bridgeIpSecretsFile + "! Not calculating any IP "
- + "address hashes in this execution!", e);
- this.persistenceProblemWithSecrets = true;
- }
- }
-
- /* If we're configured to keep secrets only for a limited time, define
- * the cut-off day and time. */
- if (limitBridgeSanitizingInterval >= 0L) {
- SimpleDateFormat formatter = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
- this.bridgeSanitizingCutOffTimestamp = formatter.format(
- System.currentTimeMillis() - 24L * 60L * 60L * 1000L
- * limitBridgeSanitizingInterval);
- } else {
- this.bridgeSanitizingCutOffTimestamp = "1999-12-31 23:59:59";
- }
- }
-
- private String scrubOrAddress(String orAddress, byte[] fingerprintBytes,
- String published) throws IOException {
- if (!orAddress.contains(":")) {
- /* Malformed or-address or a line. */
- return null;
- }
- String addressPart = orAddress.substring(0,
- orAddress.lastIndexOf(":"));
- String portPart = orAddress.substring(orAddress.lastIndexOf(":") + 1);
- String scrubbedAddressPart = null;
- if (addressPart.startsWith("[")) {
- scrubbedAddressPart = this.scrubIpv6Address(addressPart,
- fingerprintBytes, published);
- } else {
- scrubbedAddressPart = this.scrubIpv4Address(addressPart,
- fingerprintBytes, published);
- }
- return (scrubbedAddressPart == null ? null :
- scrubbedAddressPart + ":" + portPart);
- }
-
- private String scrubIpv4Address(String address, byte[] fingerprintBytes,
- String published) throws IOException {
- if (this.replaceIPAddressesWithHashes) {
- if (this.persistenceProblemWithSecrets) {
- /* There's a persistence problem, so we shouldn't scrub more IP
- * addresses in this execution. */
- return null;
- }
- byte[] hashInput = new byte[4 + 20 + 31];
- String[] ipParts = address.split("\\.");
- for (int i = 0; i < 4; i++) {
- hashInput[i] = (byte) Integer.parseInt(ipParts[i]);
- }
- System.arraycopy(fingerprintBytes, 0, hashInput, 4, 20);
- String month = published.substring(0, "yyyy-MM".length());
- byte[] secret = this.getSecretForMonth(month);
- System.arraycopy(secret, 0, hashInput, 24, 31);
- byte[] hashOutput = DigestUtils.sha256(hashInput);
- String hashedAddress = "10."
- + (((int) hashOutput[0] + 256) % 256) + "."
- + (((int) hashOutput[1] + 256) % 256) + "."
- + (((int) hashOutput[2] + 256) % 256);
- return hashedAddress;
- } else {
- return "127.0.0.1";
- }
- }
-
- private String scrubIpv6Address(String address, byte[] fingerprintBytes,
- String published) throws IOException {
- StringBuilder sb = new StringBuilder("[fd9f:2e19:3bcf::");
- if (this.replaceIPAddressesWithHashes) {
- if (this.persistenceProblemWithSecrets) {
- /* There's a persistence problem, so we shouldn't scrub more IP
- * addresses in this execution. */
- return null;
- }
- byte[] hashInput = new byte[16 + 20 + 19];
- String[] doubleColonSeparatedParts = address.substring(1,
- address.length() - 1).split("::", -1);
- if (doubleColonSeparatedParts.length > 2) {
- /* Invalid IPv6 address. */
- return null;
- }
- List<String> hexParts = new ArrayList<String>();
- for (String doubleColonSeparatedPart : doubleColonSeparatedParts) {
- StringBuilder hexPart = new StringBuilder();
- String[] parts = doubleColonSeparatedPart.split(":", -1);
- if (parts.length < 1 || parts.length > 8) {
- /* Invalid IPv6 address. */
- return null;
- }
- for (int i = 0; i < parts.length; i++) {
- String part = parts[i];
- if (part.contains(".")) {
- String[] ipParts = part.split("\\.");
- byte[] ipv4Bytes = new byte[4];
- if (ipParts.length != 4) {
- /* Invalid IPv4 part in IPv6 address. */
- return null;
- }
- for (int m = 0; m < 4; m++) {
- ipv4Bytes[m] = (byte) Integer.parseInt(ipParts[m]);
- }
- hexPart.append(Hex.encodeHexString(ipv4Bytes));
- } else if (part.length() > 4) {
- /* Invalid IPv6 address. */
- return null;
- } else {
- for (int k = part.length(); k < 4; k++) {
- hexPart.append("0");
- }
- hexPart.append(part);
- }
- }
- hexParts.add(hexPart.toString());
- }
- StringBuilder hex = new StringBuilder();
- hex.append(hexParts.get(0));
- if (hexParts.size() == 2) {
- for (int i = 32 - hexParts.get(0).length()
- - hexParts.get(1).length(); i > 0; i--) {
- hex.append("0");
- }
- hex.append(hexParts.get(1));
- }
- byte[] ipBytes = null;
- try {
- ipBytes = Hex.decodeHex(hex.toString().toCharArray());
- } catch (DecoderException e) {
- /* TODO Invalid IPv6 address. */
- return null;
- }
- if (ipBytes.length != 16) {
- /* TODO Invalid IPv6 address. */
- return null;
- }
- System.arraycopy(ipBytes, 0, hashInput, 0, 16);
- System.arraycopy(fingerprintBytes, 0, hashInput, 16, 20);
- String month = published.substring(0, "yyyy-MM".length());
- byte[] secret = this.getSecretForMonth(month);
- System.arraycopy(secret, 31, hashInput, 36, 19);
- String hashOutput = DigestUtils.sha256Hex(hashInput);
- sb.append(hashOutput.substring(hashOutput.length() - 6,
- hashOutput.length() - 4));
- sb.append(":");
- sb.append(hashOutput.substring(hashOutput.length() - 4));
- }
- sb.append("]");
- return sb.toString();
- }
-
- private byte[] getSecretForMonth(String month) throws IOException {
- if (!this.secretsForHashingIPAddresses.containsKey(month) ||
- this.secretsForHashingIPAddresses.get(month).length == 31) {
- byte[] secret = new byte[50];
- this.secureRandom.nextBytes(secret);
- if (this.secretsForHashingIPAddresses.containsKey(month)) {
- System.arraycopy(this.secretsForHashingIPAddresses.get(month), 0,
- secret, 0, 31);
- }
- if (month.compareTo(
- this.bridgeSanitizingCutOffTimestamp) < 0) {
- this.logger.warning("Generated a secret that we won't make "
- + "persistent, because it's outside our bridge descriptor "
- + "sanitizing interval.");
- } else {
- /* Append secret to file on disk immediately before using it, or
- * we might end with inconsistently sanitized bridges. */
- try {
- if (!this.bridgeIpSecretsFile.exists()) {
- this.bridgeIpSecretsFile.getParentFile().mkdirs();
- }
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.bridgeIpSecretsFile,
- this.bridgeIpSecretsFile.exists()));
- bw.write(month + "," + Hex.encodeHexString(secret) + "\n");
- bw.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not store new secret "
- + "to disk! Not calculating any IP address hashes in "
- + "this execution!", e);
- this.persistenceProblemWithSecrets = true;
- throw new IOException(e);
- }
- }
- this.secretsForHashingIPAddresses.put(month, secret);
- }
- return this.secretsForHashingIPAddresses.get(month);
- }
-
- /**
- * Sanitizes a network status and writes it to disk.
- */
- public void sanitizeAndStoreNetworkStatus(byte[] data,
- String publicationTime) {
-
- if (this.persistenceProblemWithSecrets) {
- /* There's a persistence problem, so we shouldn't scrub more IP
- * addresses in this execution. */
- return;
- }
-
- if (this.bridgeSanitizingCutOffTimestamp.
- compareTo(publicationTime) > 0) {
- this.logger.log(!this.haveWarnedAboutInterval ? Level.WARNING
- : Level.FINE, "Sanitizing and storing network status with "
- + "publication time outside our descriptor sanitizing "
- + "interval.");
- this.haveWarnedAboutInterval = true;
- }
-
- /* Parse the given network status line by line. */
- SortedMap<String, String> scrubbedLines =
- new TreeMap<String, String>();
- try {
- StringBuilder scrubbed = new StringBuilder();
- BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
- String line = null;
- String mostRecentDescPublished = null;
- byte[] fingerprintBytes = null;
- String descPublicationTime = null;
- String hashedBridgeIdentityHex = null;
- while ((line = br.readLine()) != null) {
-
- /* r lines contain sensitive information that needs to be removed
- * or replaced. */
- if (line.startsWith("r ")) {
-
- /* Clear buffer from previously scrubbed lines. */
- if (scrubbed.length() > 0) {
- String scrubbedLine = scrubbed.toString();
- scrubbedLines.put(hashedBridgeIdentityHex, scrubbedLine);
- scrubbed = new StringBuilder();
- }
-
- /* Parse the relevant parts of this r line. */
- String[] parts = line.split(" ");
- String nickname = parts[1];
- fingerprintBytes = Base64.decodeBase64(parts[2] + "==");
- String descriptorIdentifier = parts[3];
- descPublicationTime = parts[4] + " " + parts[5];
- String address = parts[6];
- String orPort = parts[7];
- String dirPort = parts[8];
-
- /* Determine most recent descriptor publication time. */
- if (descPublicationTime.compareTo(publicationTime) <= 0 &&
- (mostRecentDescPublished == null ||
- descPublicationTime.compareTo(
- mostRecentDescPublished) > 0)) {
- mostRecentDescPublished = descPublicationTime;
- }
-
- /* Write scrubbed r line to buffer. */
- byte[] hashedBridgeIdentity = DigestUtils.sha(fingerprintBytes);
- String hashedBridgeIdentityBase64 = Base64.encodeBase64String(
- hashedBridgeIdentity).substring(0, 27);
- hashedBridgeIdentityHex = Hex.encodeHexString(
- hashedBridgeIdentity);
- String hashedDescriptorIdentifier = Base64.encodeBase64String(
- DigestUtils.sha(Base64.decodeBase64(descriptorIdentifier
- + "=="))).substring(0, 27);
- String scrubbedAddress = scrubIpv4Address(address,
- fingerprintBytes,
- descPublicationTime);
- scrubbed.append("r " + nickname + " "
- + hashedBridgeIdentityBase64 + " "
- + hashedDescriptorIdentifier + " " + descPublicationTime
- + " " + scrubbedAddress + " " + orPort + " " + dirPort
- + "\n");
-
- /* Sanitize any addresses in a lines using the fingerprint and
- * descriptor publication time from the previous r line. */
- } else if (line.startsWith("a ")) {
- String scrubbedOrAddress = scrubOrAddress(
- line.substring("a ".length()), fingerprintBytes,
- descPublicationTime);
- if (scrubbedOrAddress != null) {
- scrubbed.append("a " + scrubbedOrAddress + "\n");
- } else {
- this.logger.warning("Invalid address in line '" + line
- + "' in bridge network status. Skipping line!");
- }
-
- /* Nothing special about s, w, and p lines; just copy them. */
- } else if (line.startsWith("s ") || line.equals("s") ||
- line.startsWith("w ") || line.equals("w") ||
- line.startsWith("p ") || line.equals("p")) {
- scrubbed.append(line + "\n");
-
- /* There should be nothing else but r, w, p, and s lines in the
- * network status. If there is, we should probably learn before
- * writing anything to the sanitized descriptors. */
- } else {
- this.logger.fine("Unknown line '" + line + "' in bridge "
- + "network status. Not writing to disk!");
- return;
- }
- }
- br.close();
- if (scrubbed.length() > 0) {
- String scrubbedLine = scrubbed.toString();
- scrubbedLines.put(hashedBridgeIdentityHex, scrubbedLine);
- scrubbed = new StringBuilder();
- }
-
- /* Check if we can tell from the descriptor publication times
- * whether this status is possibly stale. */
- SimpleDateFormat formatter = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (formatter.parse(publicationTime).getTime() -
- formatter.parse(mostRecentDescPublished).getTime() >
- 60L * 60L * 1000L) {
- this.logger.warning("The most recent descriptor in the bridge "
- + "network status published at " + publicationTime + " was "
- + "published at " + mostRecentDescPublished + " which is "
- + "more than 1 hour before the status. This is a sign for "
- + "the status being stale. Please check!");
- }
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not parse timestamp in "
- + "bridge network status.", e);
- return;
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not parse bridge network "
- + "status.", e);
- return;
- }
-
- /* Write the sanitized network status to disk. */
- try {
-
- /* Determine file name. */
- String syear = publicationTime.substring(0, 4);
- String smonth = publicationTime.substring(5, 7);
- String sday = publicationTime.substring(8, 10);
- String stime = publicationTime.substring(11, 13)
- + publicationTime.substring(14, 16)
- + publicationTime.substring(17, 19);
- File statusFile = new File(
- this.sanitizedBridgesDirectory.getAbsolutePath() + "/" + syear
- + "/" + smonth + "/statuses/" + sday + "/" + syear + smonth
- + sday + "-" + stime + "-"
- + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D");
-
- /* Create all parent directories to write this network status. */
- statusFile.getParentFile().mkdirs();
-
- /* Write sanitized network status to disk. */
- BufferedWriter bw = new BufferedWriter(new FileWriter(statusFile));
- bw.write("@type bridge-network-status 1.0\n");
- bw.write("published " + publicationTime + "\n");
- for (String scrubbed : scrubbedLines.values()) {
- bw.write(scrubbed);
- }
- bw.close();
-
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not write sanitized bridge "
- + "network status to disk.", e);
- return;
- }
- }
-
- /**
- * Sanitizes a bridge server descriptor and writes it to disk.
- */
- public void sanitizeAndStoreServerDescriptor(byte[] data) {
-
- if (this.persistenceProblemWithSecrets) {
- /* There's a persistence problem, so we shouldn't scrub more IP
- * addresses in this execution. */
- return;
- }
-
- /* Parse descriptor to generate a sanitized version. */
- String scrubbedDesc = null, published = null;
- try {
- BufferedReader br = new BufferedReader(new StringReader(
- new String(data, "US-ASCII")));
- StringBuilder scrubbed = new StringBuilder();
- String line = null, hashedBridgeIdentity = null, address = null,
- routerLine = null, scrubbedAddress = null;
- List<String> orAddresses = null, scrubbedOrAddresses = null;
- boolean skipCrypto = false;
- while ((line = br.readLine()) != null) {
-
- /* Skip all crypto parts that might be used to derive the bridge's
- * identity fingerprint. */
- if (skipCrypto && !line.startsWith("-----END ")) {
- continue;
-
- /* Store the router line for later processing, because we may need
- * the bridge identity fingerprint for replacing the IP address in
- * the scrubbed version. */
- } else if (line.startsWith("router ")) {
- address = line.split(" ")[2];
- routerLine = line;
-
- /* Store or-address parts in a list and sanitize them when we have
- * read the fingerprint. */
- } else if (line.startsWith("or-address ")) {
- if (orAddresses == null) {
- orAddresses = new ArrayList<String>();
- }
- orAddresses.add(line.substring("or-address ".length()));
-
- /* Parse the publication time to see if we're still inside the
- * sanitizing interval. */
- } else if (line.startsWith("published ")) {
- published = line.substring("published ".length());
- if (this.bridgeSanitizingCutOffTimestamp.
- compareTo(published) > 0) {
- this.logger.log(!this.haveWarnedAboutInterval
- ? Level.WARNING : Level.FINE, "Sanitizing and storing "
- + "server descriptor with publication time outside our "
- + "descriptor sanitizing interval.");
- this.haveWarnedAboutInterval = true;
- }
- scrubbed.append(line + "\n");
-
- /* Parse the fingerprint to determine the hashed bridge
- * identity. */
- } else if (line.startsWith("opt fingerprint ") ||
- line.startsWith("fingerprint ")) {
- String fingerprint = line.substring(line.startsWith("opt ") ?
- "opt fingerprint".length() : "fingerprint".length()).
- replaceAll(" ", "").toLowerCase();
- byte[] fingerprintBytes = Hex.decodeHex(
- fingerprint.toCharArray());
- hashedBridgeIdentity = DigestUtils.shaHex(fingerprintBytes).
- toLowerCase();
- try {
- scrubbedAddress = scrubIpv4Address(address, fingerprintBytes,
- published);
- if (orAddresses != null) {
- scrubbedOrAddresses = new ArrayList<String>();
- for (String orAddress : orAddresses) {
- String scrubbedOrAddress = scrubOrAddress(orAddress,
- fingerprintBytes, published);
- if (scrubbedOrAddress != null) {
- scrubbedOrAddresses.add(scrubbedOrAddress);
- } else {
- this.logger.warning("Invalid address in line "
- + "'or-address " + orAddress + "' in bridge server "
- + "descriptor. Skipping line!");
- }
- }
- }
- } catch (IOException e) {
- /* There's a persistence problem, so we shouldn't scrub more
- * IP addresses in this execution. */
- this.persistenceProblemWithSecrets = true;
- return;
- }
- scrubbed.append((line.startsWith("opt ") ? "opt " : "")
- + "fingerprint");
- for (int i = 0; i < hashedBridgeIdentity.length() / 4; i++)
- scrubbed.append(" " + hashedBridgeIdentity.substring(4 * i,
- 4 * (i + 1)).toUpperCase());
- scrubbed.append("\n");
-
- /* Replace the contact line (if present) with a generic one. */
- } else if (line.startsWith("contact ")) {
- scrubbed.append("contact somebody\n");
-
- /* When we reach the signature, we're done. Write the sanitized
- * descriptor to disk below. */
- } else if (line.startsWith("router-signature")) {
- String[] routerLineParts = routerLine.split(" ");
- scrubbedDesc = "router " + routerLineParts[1] + " "
- + scrubbedAddress + " " + routerLineParts[3] + " "
- + routerLineParts[4] + " " + routerLineParts[5] + "\n";
- if (scrubbedOrAddresses != null) {
- for (String scrubbedOrAddress : scrubbedOrAddresses) {
- scrubbedDesc = scrubbedDesc += "or-address "
- + scrubbedOrAddress + "\n";
- }
- }
- scrubbedDesc += scrubbed.toString();
- break;
-
- /* Replace extra-info digest with the hashed digest of the
- * non-scrubbed descriptor. */
- } else if (line.startsWith("opt extra-info-digest ") ||
- line.startsWith("extra-info-digest ")) {
- String extraInfoDescriptorIdentifier = line.substring(
- line.indexOf("extra-info-digest ")
- + "extra-info-digest ".length());
- String hashedExtraInfoDescriptorIdentifier =
- DigestUtils.shaHex(Hex.decodeHex(
- extraInfoDescriptorIdentifier.toCharArray())).toUpperCase();
- scrubbed.append((line.startsWith("opt ") ? "opt " : "")
- + "extra-info-digest " + hashedExtraInfoDescriptorIdentifier
- + "\n");
-
- /* Possibly sanitize reject lines if they contain the bridge's own
- * IP address. */
- } else if (line.startsWith("reject ")) {
- if (address != null && line.startsWith("reject " + address)) {
- scrubbed.append("reject " + scrubbedAddress
- + line.substring("reject ".length() + address.length())
- + "\n");
- } else {
- scrubbed.append(line + "\n");
- }
-
- /* Write the following lines unmodified to the sanitized
- * descriptor. */
- } else if (line.startsWith("accept ")
- || line.startsWith("platform ")
- || line.startsWith("opt protocols ")
- || line.startsWith("protocols ")
- || line.startsWith("uptime ")
- || line.startsWith("bandwidth ")
- || line.startsWith("opt hibernating ")
- || line.startsWith("hibernating ")
- || line.equals("opt hidden-service-dir")
- || line.equals("hidden-service-dir")
- || line.equals("opt caches-extra-info")
- || line.equals("caches-extra-info")
- || line.equals("opt allow-single-hop-exits")
- || line.equals("allow-single-hop-exits")) {
- scrubbed.append(line + "\n");
-
- /* Replace node fingerprints in the family line with their hashes
- * and leave nicknames unchanged. */
- } else if (line.startsWith("family ")) {
- StringBuilder familyLine = new StringBuilder("family");
- for (String s : line.substring(7).split(" ")) {
- if (s.startsWith("$")) {
- familyLine.append(" $" + DigestUtils.shaHex(Hex.decodeHex(
- s.substring(1).toCharArray())).toUpperCase());
- } else {
- familyLine.append(" " + s);
- }
- }
- scrubbed.append(familyLine.toString() + "\n");
-
- /* Skip the purpose line that the bridge authority adds to its
- * cached-descriptors file. */
- } else if (line.startsWith("@purpose ")) {
- continue;
-
- /* Skip all crypto parts that might leak the bridge's identity
- * fingerprint. */
- } else if (line.startsWith("-----BEGIN ")
- || line.equals("onion-key") || line.equals("signing-key")) {
- skipCrypto = true;
-
- /* Stop skipping lines when the crypto parts are over. */
- } else if (line.startsWith("-----END ")) {
- skipCrypto = false;
-
- /* If we encounter an unrecognized line, stop parsing and print
- * out a warning. We might have overlooked sensitive information
- * that we need to remove or replace for the sanitized descriptor
- * version. */
- } else {
- this.logger.fine("Unrecognized line '" + line + "'. Skipping.");
- return;
- }
- }
- br.close();
- } catch (Exception e) {
- this.logger.log(Level.WARNING, "Could not parse server "
- + "descriptor.", e);
- return;
- }
-
- /* Determine filename of sanitized server descriptor. */
- String descriptorDigest = null;
- try {
- String ascii = new String(data, "US-ASCII");
- String startToken = "router ";
- String sigToken = "\nrouter-signature\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- if (start >= 0 && sig >= 0 && sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- descriptorDigest = DigestUtils.shaHex(DigestUtils.sha(forDigest));
- }
- } catch (UnsupportedEncodingException e) {
- /* Handle below. */
- }
- if (descriptorDigest == null) {
- this.logger.log(Level.WARNING, "Could not calculate server "
- + "descriptor digest.");
- return;
- }
- String dyear = published.substring(0, 4);
- String dmonth = published.substring(5, 7);
- File newFile = new File(
- this.sanitizedBridgesDirectory.getAbsolutePath() + "/"
- + dyear + "/" + dmonth + "/server-descriptors/"
- + "/" + descriptorDigest.charAt(0) + "/"
- + descriptorDigest.charAt(1) + "/"
- + descriptorDigest);
-
- /* Write sanitized server descriptor to disk, including all its parent
- * directories. */
- try {
- newFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(newFile));
- bw.write("@type bridge-server-descriptor 1.0\n");
- bw.write(scrubbedDesc);
- bw.write("router-digest " + descriptorDigest.toUpperCase() + "\n");
- bw.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not write sanitized server "
- + "descriptor to disk.", e);
- return;
- }
- }
-
- /**
- * Sanitizes an extra-info descriptor and writes it to disk.
- */
- public void sanitizeAndStoreExtraInfoDescriptor(byte[] data) {
-
- /* Parse descriptor to generate a sanitized version. */
- String scrubbedDesc = null, published = null;
- try {
- BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
- String line = null;
- StringBuilder scrubbed = null;
- String hashedBridgeIdentity = null;
- while ((line = br.readLine()) != null) {
-
- /* Parse bridge identity from extra-info line and replace it with
- * its hash in the sanitized descriptor. */
- String[] parts = line.split(" ");
- if (line.startsWith("extra-info ")) {
- hashedBridgeIdentity = DigestUtils.shaHex(Hex.decodeHex(
- parts[2].toCharArray())).toLowerCase();
- scrubbed = new StringBuilder("extra-info " + parts[1] + " "
- + hashedBridgeIdentity.toUpperCase() + "\n");
-
- /* Parse the publication time to determine the file name. */
- } else if (line.startsWith("published ")) {
- scrubbed.append(line + "\n");
- published = line.substring("published ".length());
-
- /* Remove everything from transport lines except the transport
- * name. */
- } else if (line.startsWith("transport ")) {
- if (parts.length < 3) {
- this.logger.fine("Illegal line in extra-info descriptor: '"
- + line + "'. Skipping descriptor.");
- return;
- }
- scrubbed.append("transport " + parts[1] + "\n");
-
- /* Skip transport-info lines entirely. */
- } else if (line.startsWith("transport-info ")) {
-
- /* Write the following lines unmodified to the sanitized
- * descriptor. */
- } else if (line.startsWith("write-history ")
- || line.startsWith("read-history ")
- || line.startsWith("geoip-start-time ")
- || line.startsWith("geoip-client-origins ")
- || line.startsWith("geoip-db-digest ")
- || line.startsWith("conn-bi-direct ")
- || line.startsWith("bridge-")
- || line.startsWith("dirreq-")
- || line.startsWith("cell-")
- || line.startsWith("entry-")
- || line.startsWith("exit-")) {
- scrubbed.append(line + "\n");
-
- /* When we reach the signature, we're done. Write the sanitized
- * descriptor to disk below. */
- } else if (line.startsWith("router-signature")) {
- scrubbedDesc = scrubbed.toString();
- break;
-
- /* If we encounter an unrecognized line, stop parsing and print
- * out a warning. We might have overlooked sensitive information
- * that we need to remove or replace for the sanitized descriptor
- * version. */
- } else {
- this.logger.fine("Unrecognized line '" + line + "'. Skipping.");
- return;
- }
- }
- br.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not parse extra-info "
- + "descriptor.", e);
- return;
- } catch (DecoderException e) {
- this.logger.log(Level.WARNING, "Could not parse extra-info "
- + "descriptor.", e);
- return;
- }
-
- /* Determine filename of sanitized extra-info descriptor. */
- String descriptorDigest = null;
- try {
- String ascii = new String(data, "US-ASCII");
- String startToken = "extra-info ";
- String sigToken = "\nrouter-signature\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- if (start >= 0 && sig >= 0 && sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- descriptorDigest = DigestUtils.shaHex(DigestUtils.sha(forDigest));
- }
- } catch (UnsupportedEncodingException e) {
- /* Handle below. */
- }
- if (descriptorDigest == null) {
- this.logger.log(Level.WARNING, "Could not calculate extra-info "
- + "descriptor digest.");
- return;
- }
- String dyear = published.substring(0, 4);
- String dmonth = published.substring(5, 7);
- File newFile = new File(
- this.sanitizedBridgesDirectory.getAbsolutePath() + "/"
- + dyear + "/" + dmonth + "/extra-infos/"
- + descriptorDigest.charAt(0) + "/"
- + descriptorDigest.charAt(1) + "/"
- + descriptorDigest);
-
- /* Write sanitized extra-info descriptor to disk, including all its
- * parent directories. */
- try {
- newFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(newFile));
- bw.write("@type bridge-extra-info 1.1\n");
- bw.write(scrubbedDesc);
- bw.write("router-digest " + descriptorDigest.toUpperCase() + "\n");
- bw.close();
- } catch (Exception e) {
- this.logger.log(Level.WARNING, "Could not write sanitized "
- + "extra-info descriptor to disk.", e);
- }
- }
-
- /**
- * Rewrite all network statuses that might contain references to server
- * descriptors we added or updated in this execution. This applies to
- * all statuses that have been published up to 24 hours after any added
- * or updated server descriptor.
- */
- public void finishWriting() {
-
- /* Delete secrets that we don't need anymore. */
- if (!this.secretsForHashingIPAddresses.isEmpty() &&
- this.secretsForHashingIPAddresses.firstKey().compareTo(
- this.bridgeSanitizingCutOffTimestamp) < 0) {
- try {
- int kept = 0, deleted = 0;
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.bridgeIpSecretsFile));
- for (Map.Entry<String, byte[]> e :
- this.secretsForHashingIPAddresses.entrySet()) {
- if (e.getKey().compareTo(
- this.bridgeSanitizingCutOffTimestamp) < 0) {
- deleted++;
- } else {
- bw.write(e.getKey() + "," + Hex.encodeHexString(e.getValue())
- + "\n");
- kept++;
- }
- }
- bw.close();
- this.logger.info("Deleted " + deleted + " secrets that we don't "
- + "need anymore and kept " + kept + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not store reduced set of "
- + "secrets to disk! This is a bad sign, better check what's "
- + "going on!", e);
- }
- }
- }
-}
-
diff --git a/src/org/torproject/ernie/db/TorperfDownloader.java b/src/org/torproject/ernie/db/TorperfDownloader.java
deleted file mode 100644
index 058e29b..0000000
--- a/src/org/torproject/ernie/db/TorperfDownloader.java
+++ /dev/null
@@ -1,573 +0,0 @@
-/* Copyright 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.text.SimpleDateFormat;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/* Download possibly truncated Torperf .data and .extradata files from
- * configured sources, append them to the files we already have, and merge
- * the two files into the .tpf format. */
-public class TorperfDownloader {
-
- private File torperfOutputDirectory = null;
- private SortedMap<String, String> torperfSources = null;
- private List<String> torperfFilesLines = null;
- private Logger logger = null;
- private SimpleDateFormat dateFormat;
-
- public TorperfDownloader(File torperfOutputDirectory,
- SortedMap<String, String> torperfSources,
- List<String> torperfFilesLines) {
- if (torperfOutputDirectory == null) {
- throw new IllegalArgumentException();
- }
- this.torperfOutputDirectory = torperfOutputDirectory;
- this.torperfSources = torperfSources;
- this.torperfFilesLines = torperfFilesLines;
- if (!this.torperfOutputDirectory.exists()) {
- this.torperfOutputDirectory.mkdirs();
- }
- this.logger = Logger.getLogger(TorperfDownloader.class.getName());
- this.dateFormat = new SimpleDateFormat("yyyy-MM-dd");
- this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- this.readLastMergedTimestamps();
- for (String torperfFilesLine : this.torperfFilesLines) {
- this.downloadAndMergeFiles(torperfFilesLine);
- }
- this.writeLastMergedTimestamps();
- }
-
- private File torperfLastMergedFile =
- new File("stats/torperf-last-merged");
- SortedMap<String, String> lastMergedTimestamps =
- new TreeMap<String, String>();
- private void readLastMergedTimestamps() {
- if (!this.torperfLastMergedFile.exists()) {
- return;
- }
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- this.torperfLastMergedFile));
- String line;
- while ((line = br.readLine()) != null) {
- String[] parts = line.split(" ");
- String fileName = null, timestamp = null;
- if (parts.length == 2) {
- try {
- Double.parseDouble(parts[1]);
- fileName = parts[0];
- timestamp = parts[1];
- } catch (NumberFormatException e) {
- /* Handle below. */
- }
- }
- if (fileName == null || timestamp == null) {
- this.logger.log(Level.WARNING, "Invalid line '" + line + "' in "
- + this.torperfLastMergedFile.getAbsolutePath() + ". "
- + "Ignoring past history of merging .data and .extradata "
- + "files.");
- this.lastMergedTimestamps.clear();
- break;
- }
- this.lastMergedTimestamps.put(fileName, timestamp);
- }
- br.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Error while reading '"
- + this.torperfLastMergedFile.getAbsolutePath() + ". Ignoring "
- + "past history of merging .data and .extradata files.");
- this.lastMergedTimestamps.clear();
- }
- }
-
- private void writeLastMergedTimestamps() {
- try {
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.torperfLastMergedFile));
- for (Map.Entry<String, String> e :
- this.lastMergedTimestamps.entrySet()) {
- String fileName = e.getKey();
- String timestamp = e.getValue();
- bw.write(fileName + " " + timestamp + "\n");
- }
- bw.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Error while writing '"
- + this.torperfLastMergedFile.getAbsolutePath() + ". This may "
- + "result in ignoring history of merging .data and .extradata "
- + "files in the next execution.", e);
- }
- }
-
- private void downloadAndMergeFiles(String torperfFilesLine) {
- String[] parts = torperfFilesLine.split(" ");
- String sourceName = parts[1];
- int fileSize = -1;
- try {
- fileSize = Integer.parseInt(parts[2]);
- } catch (NumberFormatException e) {
- this.logger.log(Level.WARNING, "Could not parse file size in "
- + "TorperfFiles configuration line '" + torperfFilesLine
- + "'.");
- return;
- }
-
- /* Download and append the .data file. */
- String dataFileName = parts[3];
- String sourceBaseUrl = torperfSources.get(sourceName);
- String dataUrl = sourceBaseUrl + dataFileName;
- String dataOutputFileName = sourceName + "-" + dataFileName;
- File dataOutputFile = new File(torperfOutputDirectory,
- dataOutputFileName);
- boolean downloadedDataFile = this.downloadAndAppendFile(dataUrl,
- dataOutputFile, true);
-
- /* Download and append the .extradata file. */
- String extradataFileName = parts[4];
- String extradataUrl = sourceBaseUrl + extradataFileName;
- String extradataOutputFileName = sourceName + "-" + extradataFileName;
- File extradataOutputFile = new File(torperfOutputDirectory,
- extradataOutputFileName);
- boolean downloadedExtradataFile = this.downloadAndAppendFile(
- extradataUrl, extradataOutputFile, false);
-
- /* Merge both files into .tpf format. */
- if (!downloadedDataFile && !downloadedExtradataFile) {
- return;
- }
- String skipUntil = null;
- if (this.lastMergedTimestamps.containsKey(dataOutputFileName)) {
- skipUntil = this.lastMergedTimestamps.get(dataOutputFileName);
- }
- try {
- skipUntil = this.mergeFiles(dataOutputFile, extradataOutputFile,
- sourceName, fileSize, skipUntil);
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed merging " + dataOutputFile
- + " and " + extradataOutputFile + ".", e);
- }
- if (skipUntil != null) {
- this.lastMergedTimestamps.put(dataOutputFileName, skipUntil);
- }
- }
-
- private boolean downloadAndAppendFile(String url, File outputFile,
- boolean isDataFile) {
-
- /* Read an existing output file to determine which line will be the
- * first to append to it. */
- String lastTimestampLine = null;
- int linesAfterLastTimestampLine = 0;
- if (outputFile.exists() && outputFile.lastModified() >
- System.currentTimeMillis() - 330L * 60L * 1000L) {
- return false;
- } else if (outputFile.exists()) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- outputFile));
- String line;
- while ((line = br.readLine()) != null) {
- if (isDataFile || line.contains(" LAUNCH")) {
- lastTimestampLine = line;
- linesAfterLastTimestampLine = 0;
- } else {
- linesAfterLastTimestampLine++;
- }
- }
- br.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed reading '"
- + outputFile.getAbsolutePath() + "' to determine the first "
- + "line to append to it.", e);
- return false;
- }
- }
- try {
- this.logger.fine("Downloading " + (isDataFile ? ".data" :
- ".extradata") + " file from '" + url + "' and merging it into "
- + "'" + outputFile.getAbsolutePath() + "'.");
- URL u = new URL(url);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- BufferedReader br = new BufferedReader(new InputStreamReader(
- huc.getInputStream()));
- String line;
- BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile,
- true));
- boolean copyLines = lastTimestampLine == null;
- while ((line = br.readLine()) != null) {
- if (copyLines && linesAfterLastTimestampLine == 0) {
- if (isDataFile || line.contains(" LAUNCH")) {
- lastTimestampLine = line;
- }
- bw.write(line + "\n");
- } else if (copyLines && linesAfterLastTimestampLine > 0) {
- linesAfterLastTimestampLine--;
- } else if (line.equals(lastTimestampLine)) {
- copyLines = true;
- }
- }
- bw.close();
- br.close();
- if (!copyLines) {
- this.logger.warning("The last timestamp line in '"
- + outputFile.getAbsolutePath() + "' is not contained in the "
- + "new file downloaded from '" + url + "'. Cannot append "
- + "new lines without possibly leaving a gap. Skipping.");
- return false;
- }
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed downloading and/or merging '"
- + url + "'.", e);
- return false;
- }
- if (lastTimestampLine == null) {
- this.logger.warning("'" + outputFile.getAbsolutePath()
- + "' doesn't contain any timestamp lines. Unable to check "
- + "whether that file is stale or not.");
- } else {
- long lastTimestampMillis = -1L;
- if (isDataFile) {
- lastTimestampMillis = Long.parseLong(lastTimestampLine.substring(
- 0, lastTimestampLine.indexOf(" "))) * 1000L;
- } else {
- lastTimestampMillis = Long.parseLong(lastTimestampLine.substring(
- lastTimestampLine.indexOf(" LAUNCH=") + " LAUNCH=".length(),
- lastTimestampLine.indexOf(".",
- lastTimestampLine.indexOf(" LAUNCH=")))) * 1000L;
- }
- if (lastTimestampMillis < System.currentTimeMillis()
- - 330L * 60L * 1000L) {
- this.logger.warning("The last timestamp in '"
- + outputFile.getAbsolutePath() + "' is more than 5:30 hours "
- + "old: " + lastTimestampMillis);
- }
- }
- return true;
- }
-
- private String mergeFiles(File dataFile, File extradataFile,
- String source, int fileSize, String skipUntil) throws IOException {
- SortedMap<String, String> config = new TreeMap<String, String>();
- config.put("SOURCE", source);
- config.put("FILESIZE", String.valueOf(fileSize));
- if (!dataFile.exists() || !extradataFile.exists()) {
- this.logger.warning("File " + dataFile.getAbsolutePath() + " or "
- + extradataFile.getAbsolutePath() + " is missing.");
- return null;
- }
- this.logger.fine("Merging " + dataFile.getAbsolutePath() + " and "
- + extradataFile.getAbsolutePath() + " into .tpf format.");
- BufferedReader brD = new BufferedReader(new FileReader(dataFile)),
- brE = new BufferedReader(new FileReader(extradataFile));
- String lineD = brD.readLine(), lineE = brE.readLine();
- int d = 1, e = 1;
- String maxDataComplete = null, maxUsedAt = null;
- while (lineD != null) {
-
- /* Parse .data line. Every valid .data line will go into the .tpf
- * format, either with additional information from the .extradata
- * file or without it. */
- if (lineD.isEmpty()) {
- this.logger.finer("Skipping empty line " + dataFile.getName()
- + ":" + d++ + ".");
- lineD = brD.readLine();
- continue;
- }
- SortedMap<String, String> data = this.parseDataLine(lineD);
- if (data == null) {
- this.logger.finer("Skipping illegal line " + dataFile.getName()
- + ":" + d++ + " '" + lineD + "'.");
- lineD = brD.readLine();
- continue;
- }
- String dataComplete = data.get("DATACOMPLETE");
- double dataCompleteSeconds = Double.parseDouble(dataComplete);
- if (skipUntil != null && dataComplete.compareTo(skipUntil) < 0) {
- this.logger.finer("Skipping " + dataFile.getName() + ":"
- + d++ + " which we already processed before.");
- lineD = brD.readLine();
- continue;
- }
- maxDataComplete = dataComplete;
-
- /* Parse .extradata line if available and try to find the one that
- * matches the .data line. */
- SortedMap<String, String> extradata = null;
- while (lineE != null) {
- if (lineE.isEmpty()) {
- this.logger.finer("Skipping " + extradataFile.getName() + ":"
- + e++ + " which is empty.");
- lineE = brE.readLine();
- continue;
- }
- if (lineE.startsWith("BUILDTIMEOUT_SET ")) {
- this.logger.finer("Skipping " + extradataFile.getName() + ":"
- + e++ + " which is a BUILDTIMEOUT_SET line.");
- lineE = brE.readLine();
- continue;
- } else if (lineE.startsWith("ok ") ||
- lineE.startsWith("error ")) {
- this.logger.finer("Skipping " + extradataFile.getName() + ":"
- + e++ + " which is in the old format.");
- lineE = brE.readLine();
- continue;
- }
- extradata = this.parseExtradataLine(lineE);
- if (extradata == null) {
- this.logger.finer("Skipping Illegal line "
- + extradataFile.getName() + ":" + e++ + " '" + lineE
- + "'.");
- lineE = brE.readLine();
- continue;
- }
- if (!extradata.containsKey("USED_AT")) {
- this.logger.finer("Skipping " + extradataFile.getName() + ":"
- + e++ + " which doesn't contain a USED_AT element.");
- lineE = brE.readLine();
- continue;
- }
- String usedAt = extradata.get("USED_AT");
- double usedAtSeconds = Double.parseDouble(usedAt);
- if (skipUntil != null && usedAt.compareTo(skipUntil) < 0) {
- this.logger.finer("Skipping " + extradataFile.getName() + ":"
- + e++ + " which we already processed before.");
- lineE = brE.readLine();
- continue;
- }
- maxUsedAt = usedAt;
- if (Math.abs(usedAtSeconds - dataCompleteSeconds) <= 1.0) {
- this.logger.fine("Merging " + extradataFile.getName() + ":"
- + e++ + " into the current .data line.");
- lineE = brE.readLine();
- break;
- } else if (usedAtSeconds > dataCompleteSeconds) {
- this.logger.finer("Comparing " + extradataFile.getName()
- + " to the next .data line.");
- extradata = null;
- break;
- } else {
- this.logger.finer("Skipping " + extradataFile.getName() + ":"
- + e++ + " which is too old to be merged with "
- + dataFile.getName() + ":" + d + ".");
- lineE = brE.readLine();
- continue;
- }
- }
-
- /* Write output line to .tpf file. */
- SortedMap<String, String> keysAndValues =
- new TreeMap<String, String>();
- if (extradata != null) {
- keysAndValues.putAll(extradata);
- }
- keysAndValues.putAll(data);
- keysAndValues.putAll(config);
- this.logger.fine("Writing " + dataFile.getName() + ":" + d++ + ".");
- lineD = brD.readLine();
- try {
- this.writeTpfLine(source, fileSize, keysAndValues);
- } catch (IOException ex) {
- this.logger.log(Level.WARNING, "Error writing output line. "
- + "Aborting to merge " + dataFile.getName() + " and "
- + extradataFile.getName() + ".", e);
- break;
- }
- }
- brD.close();
- brE.close();
- this.writeCachedTpfLines();
- if (maxDataComplete == null) {
- return maxUsedAt;
- } else if (maxUsedAt == null) {
- return maxDataComplete;
- } else if (maxDataComplete.compareTo(maxUsedAt) > 0) {
- return maxUsedAt;
- } else {
- return maxDataComplete;
- }
- }
-
- private SortedMap<Integer, String> dataTimestamps;
- private SortedMap<String, String> parseDataLine(String line) {
- String[] parts = line.trim().split(" ");
- if (line.length() == 0 || parts.length < 20) {
- return null;
- }
- if (this.dataTimestamps == null) {
- this.dataTimestamps = new TreeMap<Integer, String>();
- this.dataTimestamps.put(0, "START");
- this.dataTimestamps.put(2, "SOCKET");
- this.dataTimestamps.put(4, "CONNECT");
- this.dataTimestamps.put(6, "NEGOTIATE");
- this.dataTimestamps.put(8, "REQUEST");
- this.dataTimestamps.put(10, "RESPONSE");
- this.dataTimestamps.put(12, "DATAREQUEST");
- this.dataTimestamps.put(14, "DATARESPONSE");
- this.dataTimestamps.put(16, "DATACOMPLETE");
- this.dataTimestamps.put(21, "DATAPERC10");
- this.dataTimestamps.put(23, "DATAPERC20");
- this.dataTimestamps.put(25, "DATAPERC30");
- this.dataTimestamps.put(27, "DATAPERC40");
- this.dataTimestamps.put(29, "DATAPERC50");
- this.dataTimestamps.put(31, "DATAPERC60");
- this.dataTimestamps.put(33, "DATAPERC70");
- this.dataTimestamps.put(35, "DATAPERC80");
- this.dataTimestamps.put(37, "DATAPERC90");
- }
- SortedMap<String, String> data = new TreeMap<String, String>();
- try {
- for (Map.Entry<Integer, String> e : this.dataTimestamps.entrySet()) {
- int i = e.getKey();
- if (parts.length > i + 1) {
- String key = e.getValue();
- String value = String.format("%s.%02d", parts[i],
- Integer.parseInt(parts[i + 1]) / 10000);
- data.put(key, value);
- }
- }
- } catch (NumberFormatException e) {
- return null;
- }
- data.put("WRITEBYTES", parts[18]);
- data.put("READBYTES", parts[19]);
- if (parts.length >= 21) {
- data.put("DIDTIMEOUT", parts[20]);
- }
- return data;
- }
-
- private SortedMap<String, String> parseExtradataLine(String line) {
- String[] parts = line.split(" ");
- SortedMap<String, String> extradata = new TreeMap<String, String>();
- String previousKey = null;
- for (String part : parts) {
- String[] keyAndValue = part.split("=", -1);
- if (keyAndValue.length == 2) {
- String key = keyAndValue[0];
- previousKey = key;
- String value = keyAndValue[1];
- if (value.contains(".") && value.lastIndexOf(".") ==
- value.length() - 2) {
- /* Make sure that all floats have two trailing digits. */
- value += "0";
- }
- extradata.put(key, value);
- } else if (keyAndValue.length == 1 && previousKey != null) {
- String value = keyAndValue[0];
- if (previousKey.equals("STREAM_FAIL_REASONS") &&
- (value.equals("MISC") || value.equals("EXITPOLICY") ||
- value.equals("RESOURCELIMIT") ||
- value.equals("RESOLVEFAILED"))) {
- extradata.put(previousKey, extradata.get(previousKey) + ":"
- + value);
- } else {
- return null;
- }
- } else {
- return null;
- }
- }
- return extradata;
- }
-
- private String cachedSource;
- private int cachedFileSize;
- private String cachedStartDate;
- private SortedMap<String, String> cachedTpfLines;
- private void writeTpfLine(String source, int fileSize,
- SortedMap<String, String> keysAndValues) throws IOException {
- StringBuilder sb = new StringBuilder();
- int written = 0;
- for (Map.Entry<String, String> keyAndValue :
- keysAndValues.entrySet()) {
- String key = keyAndValue.getKey();
- String value = keyAndValue.getValue();
- sb.append((written++ > 0 ? " " : "") + key + "=" + value);
- }
- String line = sb.toString();
- String startString = keysAndValues.get("START");
- long startMillis = Long.parseLong(startString.substring(0,
- startString.indexOf("."))) * 1000L;
- String startDate = dateFormat.format(startMillis);
- if (this.cachedTpfLines == null || !source.equals(this.cachedSource) ||
- fileSize != this.cachedFileSize ||
- !startDate.equals(this.cachedStartDate)) {
- this.writeCachedTpfLines();
- this.readTpfLinesToCache(source, fileSize, startDate);
- }
- if (!this.cachedTpfLines.containsKey(startString) ||
- line.length() > this.cachedTpfLines.get(startString).length()) {
- this.cachedTpfLines.put(startString, line);
- }
- }
-
- private void readTpfLinesToCache(String source, int fileSize,
- String startDate) throws IOException {
- this.cachedTpfLines = new TreeMap<String, String>();
- this.cachedSource = source;
- this.cachedFileSize = fileSize;
- this.cachedStartDate = startDate;
- File tpfFile = new File(torperfOutputDirectory,
- startDate.replaceAll("-", "/") + "/"
- + source + "-" + String.valueOf(fileSize) + "-" + startDate
- + ".tpf");
- if (!tpfFile.exists()) {
- return;
- }
- BufferedReader br = new BufferedReader(new FileReader(tpfFile));
- String line;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("@type ")) {
- continue;
- }
- if (line.contains("START=")) {
- String startString = line.substring(line.indexOf("START=")
- + "START=".length()).split(" ")[0];
- this.cachedTpfLines.put(startString, line);
- }
- }
- br.close();
- }
-
- private void writeCachedTpfLines() throws IOException {
- if (this.cachedSource == null || this.cachedFileSize == 0 ||
- this.cachedStartDate == null || this.cachedTpfLines == null) {
- return;
- }
- File tpfFile = new File(torperfOutputDirectory,
- this.cachedStartDate.replaceAll("-", "/")
- + "/" + this.cachedSource + "-"
- + String.valueOf(this.cachedFileSize) + "-"
- + this.cachedStartDate + ".tpf");
- tpfFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(tpfFile));
- bw.write("@type torperf 1.0\n");
- for (String line : this.cachedTpfLines.values()) {
- bw.write(line + "\n");
- }
- bw.close();
- this.cachedSource = null;
- this.cachedFileSize = 0;
- this.cachedStartDate = null;
- this.cachedTpfLines = null;
- }
-}
-
diff --git a/src/org/torproject/ernie/db/bridgedescs/BridgeDescriptorParser.java b/src/org/torproject/ernie/db/bridgedescs/BridgeDescriptorParser.java
new file mode 100644
index 0000000..f0b617a
--- /dev/null
+++ b/src/org/torproject/ernie/db/bridgedescs/BridgeDescriptorParser.java
@@ -0,0 +1,46 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.bridgedescs;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class BridgeDescriptorParser {
+ private SanitizedBridgesWriter sbw;
+ private Logger logger;
+ public BridgeDescriptorParser(SanitizedBridgesWriter sbw) {
+ this.sbw = sbw;
+ this.logger =
+ Logger.getLogger(BridgeDescriptorParser.class.getName());
+ }
+ public void parse(byte[] allData, String dateTime) {
+ try {
+ BufferedReader br = new BufferedReader(new StringReader(
+ new String(allData, "US-ASCII")));
+ String line = br.readLine();
+ if (line == null) {
+ return;
+ } else if (line.startsWith("r ")) {
+ if (this.sbw != null) {
+ this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime);
+ }
+ } else if (line.startsWith("router ")) {
+ if (this.sbw != null) {
+ this.sbw.sanitizeAndStoreServerDescriptor(allData);
+ }
+ } else if (line.startsWith("extra-info ")) {
+ if (this.sbw != null) {
+ this.sbw.sanitizeAndStoreExtraInfoDescriptor(allData);
+ }
+ }
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not parse bridge descriptor.",
+ e);
+ return;
+ }
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/bridgedescs/BridgeSnapshotReader.java b/src/org/torproject/ernie/db/bridgedescs/BridgeSnapshotReader.java
new file mode 100644
index 0000000..783775c
--- /dev/null
+++ b/src/org/torproject/ernie/db/bridgedescs/BridgeSnapshotReader.java
@@ -0,0 +1,220 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.bridgedescs;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.Stack;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+
+/**
+ * Reads the half-hourly snapshots of bridge descriptors from Tonga.
+ */
+public class BridgeSnapshotReader {
+ public BridgeSnapshotReader(BridgeDescriptorParser bdp,
+ File bridgeDirectoriesDir, File statsDirectory) {
+
+ if (bdp == null || bridgeDirectoriesDir == null ||
+ statsDirectory == null) {
+ throw new IllegalArgumentException();
+ }
+
+ Logger logger =
+ Logger.getLogger(BridgeSnapshotReader.class.getName());
+ SortedSet<String> parsed = new TreeSet<String>();
+ File bdDir = bridgeDirectoriesDir;
+ File pbdFile = new File(statsDirectory, "parsed-bridge-directories");
+ boolean modified = false;
+ if (bdDir.exists()) {
+ if (pbdFile.exists()) {
+ logger.fine("Reading file " + pbdFile.getAbsolutePath() + "...");
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(pbdFile));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ parsed.add(line);
+ }
+ br.close();
+ logger.fine("Finished reading file "
+ + pbdFile.getAbsolutePath() + ".");
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed reading file "
+ + pbdFile.getAbsolutePath() + "!", e);
+ return;
+ }
+ }
+ logger.fine("Importing files in directory " + bridgeDirectoriesDir
+ + "/...");
+ Set<String> descriptorImportHistory = new HashSet<String>();
+ int parsedFiles = 0, skippedFiles = 0, parsedStatuses = 0,
+ parsedServerDescriptors = 0, skippedServerDescriptors = 0,
+ parsedExtraInfoDescriptors = 0, skippedExtraInfoDescriptors = 0;
+ Stack<File> filesInInputDir = new Stack<File>();
+ filesInInputDir.add(bdDir);
+ while (!filesInInputDir.isEmpty()) {
+ File pop = filesInInputDir.pop();
+ if (pop.isDirectory()) {
+ for (File f : pop.listFiles()) {
+ filesInInputDir.add(f);
+ }
+ } else if (!parsed.contains(pop.getName())) {
+ try {
+ FileInputStream in = new FileInputStream(pop);
+ if (in.available() > 0) {
+ TarArchiveInputStream tais = null;
+ if (pop.getName().endsWith(".tar.gz")) {
+ GzipCompressorInputStream gcis =
+ new GzipCompressorInputStream(in);
+ tais = new TarArchiveInputStream(gcis);
+ } else if (pop.getName().endsWith(".tar")) {
+ tais = new TarArchiveInputStream(in);
+ } else {
+ continue;
+ }
+ BufferedInputStream bis = new BufferedInputStream(tais);
+ String fn = pop.getName();
+ String dateTime = fn.substring(11, 21) + " "
+ + fn.substring(22, 24) + ":" + fn.substring(24, 26)
+ + ":" + fn.substring(26, 28);
+ while ((tais.getNextTarEntry()) != null) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ byte[] allData = baos.toByteArray();
+ if (allData.length == 0) {
+ continue;
+ }
+ String fileDigest = Hex.encodeHexString(DigestUtils.sha(
+ allData));
+ String ascii = new String(allData, "US-ASCII");
+ BufferedReader br3 = new BufferedReader(new StringReader(
+ ascii));
+ String firstLine = null;
+ while ((firstLine = br3.readLine()) != null) {
+ if (firstLine.startsWith("@")) {
+ continue;
+ } else {
+ break;
+ }
+ }
+ if (firstLine.startsWith("r ")) {
+ bdp.parse(allData, dateTime);
+ parsedStatuses++;
+ } else if (descriptorImportHistory.contains(fileDigest)) {
+ /* Skip server descriptors or extra-info descriptors if
+ * we parsed them before. */
+ skippedFiles++;
+ continue;
+ } else {
+ int start = -1, sig = -1, end = -1;
+ String startToken =
+ firstLine.startsWith("router ") ?
+ "router " : "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ String endToken = "\n-----END SIGNATURE-----\n";
+ while (end < ascii.length()) {
+ start = ascii.indexOf(startToken, end);
+ if (start < 0) {
+ break;
+ }
+ sig = ascii.indexOf(sigToken, start);
+ if (sig < 0) {
+ break;
+ }
+ sig += sigToken.length();
+ end = ascii.indexOf(endToken, sig);
+ if (end < 0) {
+ break;
+ }
+ end += endToken.length();
+ byte[] descBytes = new byte[end - start];
+ System.arraycopy(allData, start, descBytes, 0,
+ end - start);
+ String descriptorDigest = Hex.encodeHexString(
+ DigestUtils.sha(descBytes));
+ if (!descriptorImportHistory.contains(
+ descriptorDigest)) {
+ bdp.parse(descBytes, dateTime);
+ descriptorImportHistory.add(descriptorDigest);
+ if (firstLine.startsWith("router ")) {
+ parsedServerDescriptors++;
+ } else {
+ parsedExtraInfoDescriptors++;
+ }
+ } else {
+ if (firstLine.startsWith("router ")) {
+ skippedServerDescriptors++;
+ } else {
+ skippedExtraInfoDescriptors++;
+ }
+ }
+ }
+ }
+ descriptorImportHistory.add(fileDigest);
+ parsedFiles++;
+ }
+ bis.close();
+ }
+ in.close();
+
+ /* Let's give some memory back, or we'll run out of it. */
+ System.gc();
+
+ parsed.add(pop.getName());
+ modified = true;
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not parse bridge snapshot "
+ + pop.getName() + "!", e);
+ continue;
+ }
+ }
+ }
+ logger.fine("Finished importing files in directory "
+ + bridgeDirectoriesDir + "/. In total, we parsed "
+ + parsedFiles + " files (skipped " + skippedFiles
+ + ") containing " + parsedStatuses + " statuses, "
+ + parsedServerDescriptors + " server descriptors (skipped "
+ + skippedServerDescriptors + "), and "
+ + parsedExtraInfoDescriptors + " extra-info descriptors "
+ + "(skipped " + skippedExtraInfoDescriptors + ").");
+ if (!parsed.isEmpty() && modified) {
+ logger.fine("Writing file " + pbdFile.getAbsolutePath() + "...");
+ try {
+ pbdFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(pbdFile));
+ for (String f : parsed) {
+ bw.append(f + "\n");
+ }
+ bw.close();
+ logger.fine("Finished writing file " + pbdFile.getAbsolutePath()
+ + ".");
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed writing file "
+ + pbdFile.getAbsolutePath() + "!", e);
+ }
+ }
+ }
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
new file mode 100644
index 0000000..ec7ad4b
--- /dev/null
+++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
@@ -0,0 +1,911 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.bridgedescs;
+
+import java.io.*;
+import java.security.*;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.logging.*;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.digest.*;
+import org.apache.commons.codec.binary.*;
+
+/**
+ * Sanitizes bridge descriptors, i.e., removes all possibly sensitive
+ * information from them, and writes them to a local directory structure.
+ * During the sanitizing process, all information about the bridge
+ * identity or IP address are removed or replaced. The goal is to keep the
+ * sanitized bridge descriptors useful for statistical analysis while not
+ * making it easier for an adversary to enumerate bridges.
+ *
+ * There are three types of bridge descriptors: bridge network statuses
+ * (lists of all bridges at a given time), server descriptors (published
+ * by the bridge to advertise their capabilities), and extra-info
+ * descriptors (published by the bridge, mainly for statistical analysis).
+ */
+public class SanitizedBridgesWriter {
+
+ /**
+ * Logger for this class.
+ */
+ private Logger logger;
+
+ /**
+ * Output directory for writing sanitized bridge descriptors.
+ */
+ private File sanitizedBridgesDirectory;
+
+ private boolean replaceIPAddressesWithHashes;
+
+ private boolean persistenceProblemWithSecrets;
+
+ private SortedMap<String, byte[]> secretsForHashingIPAddresses;
+
+ private String bridgeSanitizingCutOffTimestamp;
+
+ private boolean haveWarnedAboutInterval;
+
+ private File bridgeIpSecretsFile;
+
+ private SecureRandom secureRandom;
+
+ /**
+ * Initializes this class.
+ */
+ public SanitizedBridgesWriter(File sanitizedBridgesDirectory,
+ File statsDirectory, boolean replaceIPAddressesWithHashes,
+ long limitBridgeSanitizingInterval) {
+
+ if (sanitizedBridgesDirectory == null || statsDirectory == null) {
+ throw new IllegalArgumentException();
+ }
+
+ /* Memorize argument values. */
+ this.sanitizedBridgesDirectory = sanitizedBridgesDirectory;
+ this.replaceIPAddressesWithHashes = replaceIPAddressesWithHashes;
+
+ /* Initialize logger. */
+ this.logger = Logger.getLogger(
+ SanitizedBridgesWriter.class.getName());
+
+ /* Initialize secure random number generator if we need it. */
+ if (this.replaceIPAddressesWithHashes) {
+ try {
+ this.secureRandom = SecureRandom.getInstance("SHA1PRNG", "SUN");
+ } catch (GeneralSecurityException e) {
+ this.logger.log(Level.WARNING, "Could not initialize secure "
+ + "random number generator! Not calculating any IP address "
+ + "hashes in this execution!", e);
+ this.persistenceProblemWithSecrets = true;
+ }
+ }
+
+ /* Read hex-encoded secrets for replacing IP addresses with hashes
+ * from disk. */
+ this.secretsForHashingIPAddresses = new TreeMap<String, byte[]>();
+ this.bridgeIpSecretsFile = new File(statsDirectory,
+ "bridge-ip-secrets");
+ if (this.bridgeIpSecretsFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.bridgeIpSecretsFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(",");
+ if ((line.length() != ("yyyy-MM,".length() + 31 * 2) &&
+ line.length() != ("yyyy-MM,".length() + 50 * 2)) ||
+ parts.length != 2) {
+ this.logger.warning("Invalid line in bridge-ip-secrets file "
+ + "starting with '" + line.substring(0, 7) + "'! "
+ + "Not calculating any IP address hashes in this "
+ + "execution!");
+ this.persistenceProblemWithSecrets = true;
+ break;
+ }
+ String month = parts[0];
+ byte[] secret = Hex.decodeHex(parts[1].toCharArray());
+ this.secretsForHashingIPAddresses.put(month, secret);
+ }
+ br.close();
+ if (!this.persistenceProblemWithSecrets) {
+ this.logger.fine("Read "
+ + this.secretsForHashingIPAddresses.size() + " secrets for "
+ + "hashing bridge IP addresses.");
+ }
+ } catch (DecoderException e) {
+ this.logger.log(Level.WARNING, "Failed to decode hex string in "
+ + this.bridgeIpSecretsFile + "! Not calculating any IP "
+ + "address hashes in this execution!", e);
+ this.persistenceProblemWithSecrets = true;
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed to read "
+ + this.bridgeIpSecretsFile + "! Not calculating any IP "
+ + "address hashes in this execution!", e);
+ this.persistenceProblemWithSecrets = true;
+ }
+ }
+
+ /* If we're configured to keep secrets only for a limited time, define
+ * the cut-off day and time. */
+ if (limitBridgeSanitizingInterval >= 0L) {
+ SimpleDateFormat formatter = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.bridgeSanitizingCutOffTimestamp = formatter.format(
+ System.currentTimeMillis() - 24L * 60L * 60L * 1000L
+ * limitBridgeSanitizingInterval);
+ } else {
+ this.bridgeSanitizingCutOffTimestamp = "1999-12-31 23:59:59";
+ }
+ }
+
+ private String scrubOrAddress(String orAddress, byte[] fingerprintBytes,
+ String published) throws IOException {
+ if (!orAddress.contains(":")) {
+ /* Malformed or-address or a line. */
+ return null;
+ }
+ String addressPart = orAddress.substring(0,
+ orAddress.lastIndexOf(":"));
+ String portPart = orAddress.substring(orAddress.lastIndexOf(":") + 1);
+ String scrubbedAddressPart = null;
+ if (addressPart.startsWith("[")) {
+ scrubbedAddressPart = this.scrubIpv6Address(addressPart,
+ fingerprintBytes, published);
+ } else {
+ scrubbedAddressPart = this.scrubIpv4Address(addressPart,
+ fingerprintBytes, published);
+ }
+ return (scrubbedAddressPart == null ? null :
+ scrubbedAddressPart + ":" + portPart);
+ }
+
+ private String scrubIpv4Address(String address, byte[] fingerprintBytes,
+ String published) throws IOException {
+ if (this.replaceIPAddressesWithHashes) {
+ if (this.persistenceProblemWithSecrets) {
+ /* There's a persistence problem, so we shouldn't scrub more IP
+ * addresses in this execution. */
+ return null;
+ }
+ byte[] hashInput = new byte[4 + 20 + 31];
+ String[] ipParts = address.split("\\.");
+ for (int i = 0; i < 4; i++) {
+ hashInput[i] = (byte) Integer.parseInt(ipParts[i]);
+ }
+ System.arraycopy(fingerprintBytes, 0, hashInput, 4, 20);
+ String month = published.substring(0, "yyyy-MM".length());
+ byte[] secret = this.getSecretForMonth(month);
+ System.arraycopy(secret, 0, hashInput, 24, 31);
+ byte[] hashOutput = DigestUtils.sha256(hashInput);
+ String hashedAddress = "10."
+ + (((int) hashOutput[0] + 256) % 256) + "."
+ + (((int) hashOutput[1] + 256) % 256) + "."
+ + (((int) hashOutput[2] + 256) % 256);
+ return hashedAddress;
+ } else {
+ return "127.0.0.1";
+ }
+ }
+
+ private String scrubIpv6Address(String address, byte[] fingerprintBytes,
+ String published) throws IOException {
+ StringBuilder sb = new StringBuilder("[fd9f:2e19:3bcf::");
+ if (this.replaceIPAddressesWithHashes) {
+ if (this.persistenceProblemWithSecrets) {
+ /* There's a persistence problem, so we shouldn't scrub more IP
+ * addresses in this execution. */
+ return null;
+ }
+ byte[] hashInput = new byte[16 + 20 + 19];
+ String[] doubleColonSeparatedParts = address.substring(1,
+ address.length() - 1).split("::", -1);
+ if (doubleColonSeparatedParts.length > 2) {
+ /* Invalid IPv6 address. */
+ return null;
+ }
+ List<String> hexParts = new ArrayList<String>();
+ for (String doubleColonSeparatedPart : doubleColonSeparatedParts) {
+ StringBuilder hexPart = new StringBuilder();
+ String[] parts = doubleColonSeparatedPart.split(":", -1);
+ if (parts.length < 1 || parts.length > 8) {
+ /* Invalid IPv6 address. */
+ return null;
+ }
+ for (int i = 0; i < parts.length; i++) {
+ String part = parts[i];
+ if (part.contains(".")) {
+ String[] ipParts = part.split("\\.");
+ byte[] ipv4Bytes = new byte[4];
+ if (ipParts.length != 4) {
+ /* Invalid IPv4 part in IPv6 address. */
+ return null;
+ }
+ for (int m = 0; m < 4; m++) {
+ ipv4Bytes[m] = (byte) Integer.parseInt(ipParts[m]);
+ }
+ hexPart.append(Hex.encodeHexString(ipv4Bytes));
+ } else if (part.length() > 4) {
+ /* Invalid IPv6 address. */
+ return null;
+ } else {
+ for (int k = part.length(); k < 4; k++) {
+ hexPart.append("0");
+ }
+ hexPart.append(part);
+ }
+ }
+ hexParts.add(hexPart.toString());
+ }
+ StringBuilder hex = new StringBuilder();
+ hex.append(hexParts.get(0));
+ if (hexParts.size() == 2) {
+ for (int i = 32 - hexParts.get(0).length()
+ - hexParts.get(1).length(); i > 0; i--) {
+ hex.append("0");
+ }
+ hex.append(hexParts.get(1));
+ }
+ byte[] ipBytes = null;
+ try {
+ ipBytes = Hex.decodeHex(hex.toString().toCharArray());
+ } catch (DecoderException e) {
+ /* TODO Invalid IPv6 address. */
+ return null;
+ }
+ if (ipBytes.length != 16) {
+ /* TODO Invalid IPv6 address. */
+ return null;
+ }
+ System.arraycopy(ipBytes, 0, hashInput, 0, 16);
+ System.arraycopy(fingerprintBytes, 0, hashInput, 16, 20);
+ String month = published.substring(0, "yyyy-MM".length());
+ byte[] secret = this.getSecretForMonth(month);
+ System.arraycopy(secret, 31, hashInput, 36, 19);
+ String hashOutput = DigestUtils.sha256Hex(hashInput);
+ sb.append(hashOutput.substring(hashOutput.length() - 6,
+ hashOutput.length() - 4));
+ sb.append(":");
+ sb.append(hashOutput.substring(hashOutput.length() - 4));
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
+ private byte[] getSecretForMonth(String month) throws IOException {
+ if (!this.secretsForHashingIPAddresses.containsKey(month) ||
+ this.secretsForHashingIPAddresses.get(month).length == 31) {
+ byte[] secret = new byte[50];
+ this.secureRandom.nextBytes(secret);
+ if (this.secretsForHashingIPAddresses.containsKey(month)) {
+ System.arraycopy(this.secretsForHashingIPAddresses.get(month), 0,
+ secret, 0, 31);
+ }
+ if (month.compareTo(
+ this.bridgeSanitizingCutOffTimestamp) < 0) {
+ this.logger.warning("Generated a secret that we won't make "
+ + "persistent, because it's outside our bridge descriptor "
+ + "sanitizing interval.");
+ } else {
+ /* Append secret to file on disk immediately before using it, or
+ * we might end with inconsistently sanitized bridges. */
+ try {
+ if (!this.bridgeIpSecretsFile.exists()) {
+ this.bridgeIpSecretsFile.getParentFile().mkdirs();
+ }
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.bridgeIpSecretsFile,
+ this.bridgeIpSecretsFile.exists()));
+ bw.write(month + "," + Hex.encodeHexString(secret) + "\n");
+ bw.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not store new secret "
+ + "to disk! Not calculating any IP address hashes in "
+ + "this execution!", e);
+ this.persistenceProblemWithSecrets = true;
+ throw new IOException(e);
+ }
+ }
+ this.secretsForHashingIPAddresses.put(month, secret);
+ }
+ return this.secretsForHashingIPAddresses.get(month);
+ }
+
+ /**
+ * Sanitizes a network status and writes it to disk.
+ */
+ public void sanitizeAndStoreNetworkStatus(byte[] data,
+ String publicationTime) {
+
+ if (this.persistenceProblemWithSecrets) {
+ /* There's a persistence problem, so we shouldn't scrub more IP
+ * addresses in this execution. */
+ return;
+ }
+
+ if (this.bridgeSanitizingCutOffTimestamp.
+ compareTo(publicationTime) > 0) {
+ this.logger.log(!this.haveWarnedAboutInterval ? Level.WARNING
+ : Level.FINE, "Sanitizing and storing network status with "
+ + "publication time outside our descriptor sanitizing "
+ + "interval.");
+ this.haveWarnedAboutInterval = true;
+ }
+
+ /* Parse the given network status line by line. */
+ SortedMap<String, String> scrubbedLines =
+ new TreeMap<String, String>();
+ try {
+ StringBuilder scrubbed = new StringBuilder();
+ BufferedReader br = new BufferedReader(new StringReader(new String(
+ data, "US-ASCII")));
+ String line = null;
+ String mostRecentDescPublished = null;
+ byte[] fingerprintBytes = null;
+ String descPublicationTime = null;
+ String hashedBridgeIdentityHex = null;
+ while ((line = br.readLine()) != null) {
+
+ /* r lines contain sensitive information that needs to be removed
+ * or replaced. */
+ if (line.startsWith("r ")) {
+
+ /* Clear buffer from previously scrubbed lines. */
+ if (scrubbed.length() > 0) {
+ String scrubbedLine = scrubbed.toString();
+ scrubbedLines.put(hashedBridgeIdentityHex, scrubbedLine);
+ scrubbed = new StringBuilder();
+ }
+
+ /* Parse the relevant parts of this r line. */
+ String[] parts = line.split(" ");
+ String nickname = parts[1];
+ fingerprintBytes = Base64.decodeBase64(parts[2] + "==");
+ String descriptorIdentifier = parts[3];
+ descPublicationTime = parts[4] + " " + parts[5];
+ String address = parts[6];
+ String orPort = parts[7];
+ String dirPort = parts[8];
+
+ /* Determine most recent descriptor publication time. */
+ if (descPublicationTime.compareTo(publicationTime) <= 0 &&
+ (mostRecentDescPublished == null ||
+ descPublicationTime.compareTo(
+ mostRecentDescPublished) > 0)) {
+ mostRecentDescPublished = descPublicationTime;
+ }
+
+ /* Write scrubbed r line to buffer. */
+ byte[] hashedBridgeIdentity = DigestUtils.sha(fingerprintBytes);
+ String hashedBridgeIdentityBase64 = Base64.encodeBase64String(
+ hashedBridgeIdentity).substring(0, 27);
+ hashedBridgeIdentityHex = Hex.encodeHexString(
+ hashedBridgeIdentity);
+ String hashedDescriptorIdentifier = Base64.encodeBase64String(
+ DigestUtils.sha(Base64.decodeBase64(descriptorIdentifier
+ + "=="))).substring(0, 27);
+ String scrubbedAddress = scrubIpv4Address(address,
+ fingerprintBytes,
+ descPublicationTime);
+ scrubbed.append("r " + nickname + " "
+ + hashedBridgeIdentityBase64 + " "
+ + hashedDescriptorIdentifier + " " + descPublicationTime
+ + " " + scrubbedAddress + " " + orPort + " " + dirPort
+ + "\n");
+
+ /* Sanitize any addresses in a lines using the fingerprint and
+ * descriptor publication time from the previous r line. */
+ } else if (line.startsWith("a ")) {
+ String scrubbedOrAddress = scrubOrAddress(
+ line.substring("a ".length()), fingerprintBytes,
+ descPublicationTime);
+ if (scrubbedOrAddress != null) {
+ scrubbed.append("a " + scrubbedOrAddress + "\n");
+ } else {
+ this.logger.warning("Invalid address in line '" + line
+ + "' in bridge network status. Skipping line!");
+ }
+
+ /* Nothing special about s, w, and p lines; just copy them. */
+ } else if (line.startsWith("s ") || line.equals("s") ||
+ line.startsWith("w ") || line.equals("w") ||
+ line.startsWith("p ") || line.equals("p")) {
+ scrubbed.append(line + "\n");
+
+ /* There should be nothing else but r, w, p, and s lines in the
+ * network status. If there is, we should probably learn before
+ * writing anything to the sanitized descriptors. */
+ } else {
+ this.logger.fine("Unknown line '" + line + "' in bridge "
+ + "network status. Not writing to disk!");
+ return;
+ }
+ }
+ br.close();
+ if (scrubbed.length() > 0) {
+ String scrubbedLine = scrubbed.toString();
+ scrubbedLines.put(hashedBridgeIdentityHex, scrubbedLine);
+ scrubbed = new StringBuilder();
+ }
+
+ /* Check if we can tell from the descriptor publication times
+ * whether this status is possibly stale. */
+ SimpleDateFormat formatter = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
+ if (formatter.parse(publicationTime).getTime() -
+ formatter.parse(mostRecentDescPublished).getTime() >
+ 60L * 60L * 1000L) {
+ this.logger.warning("The most recent descriptor in the bridge "
+ + "network status published at " + publicationTime + " was "
+ + "published at " + mostRecentDescPublished + " which is "
+ + "more than 1 hour before the status. This is a sign for "
+ + "the status being stale. Please check!");
+ }
+ } catch (ParseException e) {
+ this.logger.log(Level.WARNING, "Could not parse timestamp in "
+ + "bridge network status.", e);
+ return;
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not parse bridge network "
+ + "status.", e);
+ return;
+ }
+
+ /* Write the sanitized network status to disk. */
+ try {
+
+ /* Determine file name. */
+ String syear = publicationTime.substring(0, 4);
+ String smonth = publicationTime.substring(5, 7);
+ String sday = publicationTime.substring(8, 10);
+ String stime = publicationTime.substring(11, 13)
+ + publicationTime.substring(14, 16)
+ + publicationTime.substring(17, 19);
+ File statusFile = new File(
+ this.sanitizedBridgesDirectory.getAbsolutePath() + "/" + syear
+ + "/" + smonth + "/statuses/" + sday + "/" + syear + smonth
+ + sday + "-" + stime + "-"
+ + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D");
+
+ /* Create all parent directories to write this network status. */
+ statusFile.getParentFile().mkdirs();
+
+ /* Write sanitized network status to disk. */
+ BufferedWriter bw = new BufferedWriter(new FileWriter(statusFile));
+ bw.write("@type bridge-network-status 1.0\n");
+ bw.write("published " + publicationTime + "\n");
+ for (String scrubbed : scrubbedLines.values()) {
+ bw.write(scrubbed);
+ }
+ bw.close();
+
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not write sanitized bridge "
+ + "network status to disk.", e);
+ return;
+ }
+ }
+
+ /**
+ * Sanitizes a bridge server descriptor and writes it to disk.
+ */
+ public void sanitizeAndStoreServerDescriptor(byte[] data) {
+
+ if (this.persistenceProblemWithSecrets) {
+ /* There's a persistence problem, so we shouldn't scrub more IP
+ * addresses in this execution. */
+ return;
+ }
+
+ /* Parse descriptor to generate a sanitized version. */
+ String scrubbedDesc = null, published = null;
+ try {
+ BufferedReader br = new BufferedReader(new StringReader(
+ new String(data, "US-ASCII")));
+ StringBuilder scrubbed = new StringBuilder();
+ String line = null, hashedBridgeIdentity = null, address = null,
+ routerLine = null, scrubbedAddress = null;
+ List<String> orAddresses = null, scrubbedOrAddresses = null;
+ boolean skipCrypto = false;
+ while ((line = br.readLine()) != null) {
+
+ /* Skip all crypto parts that might be used to derive the bridge's
+ * identity fingerprint. */
+ if (skipCrypto && !line.startsWith("-----END ")) {
+ continue;
+
+ /* Store the router line for later processing, because we may need
+ * the bridge identity fingerprint for replacing the IP address in
+ * the scrubbed version. */
+ } else if (line.startsWith("router ")) {
+ address = line.split(" ")[2];
+ routerLine = line;
+
+ /* Store or-address parts in a list and sanitize them when we have
+ * read the fingerprint. */
+ } else if (line.startsWith("or-address ")) {
+ if (orAddresses == null) {
+ orAddresses = new ArrayList<String>();
+ }
+ orAddresses.add(line.substring("or-address ".length()));
+
+ /* Parse the publication time to see if we're still inside the
+ * sanitizing interval. */
+ } else if (line.startsWith("published ")) {
+ published = line.substring("published ".length());
+ if (this.bridgeSanitizingCutOffTimestamp.
+ compareTo(published) > 0) {
+ this.logger.log(!this.haveWarnedAboutInterval
+ ? Level.WARNING : Level.FINE, "Sanitizing and storing "
+ + "server descriptor with publication time outside our "
+ + "descriptor sanitizing interval.");
+ this.haveWarnedAboutInterval = true;
+ }
+ scrubbed.append(line + "\n");
+
+ /* Parse the fingerprint to determine the hashed bridge
+ * identity. */
+ } else if (line.startsWith("opt fingerprint ") ||
+ line.startsWith("fingerprint ")) {
+ String fingerprint = line.substring(line.startsWith("opt ") ?
+ "opt fingerprint".length() : "fingerprint".length()).
+ replaceAll(" ", "").toLowerCase();
+ byte[] fingerprintBytes = Hex.decodeHex(
+ fingerprint.toCharArray());
+ hashedBridgeIdentity = DigestUtils.shaHex(fingerprintBytes).
+ toLowerCase();
+ try {
+ scrubbedAddress = scrubIpv4Address(address, fingerprintBytes,
+ published);
+ if (orAddresses != null) {
+ scrubbedOrAddresses = new ArrayList<String>();
+ for (String orAddress : orAddresses) {
+ String scrubbedOrAddress = scrubOrAddress(orAddress,
+ fingerprintBytes, published);
+ if (scrubbedOrAddress != null) {
+ scrubbedOrAddresses.add(scrubbedOrAddress);
+ } else {
+ this.logger.warning("Invalid address in line "
+ + "'or-address " + orAddress + "' in bridge server "
+ + "descriptor. Skipping line!");
+ }
+ }
+ }
+ } catch (IOException e) {
+ /* There's a persistence problem, so we shouldn't scrub more
+ * IP addresses in this execution. */
+ this.persistenceProblemWithSecrets = true;
+ return;
+ }
+ scrubbed.append((line.startsWith("opt ") ? "opt " : "")
+ + "fingerprint");
+ for (int i = 0; i < hashedBridgeIdentity.length() / 4; i++)
+ scrubbed.append(" " + hashedBridgeIdentity.substring(4 * i,
+ 4 * (i + 1)).toUpperCase());
+ scrubbed.append("\n");
+
+ /* Replace the contact line (if present) with a generic one. */
+ } else if (line.startsWith("contact ")) {
+ scrubbed.append("contact somebody\n");
+
+ /* When we reach the signature, we're done. Write the sanitized
+ * descriptor to disk below. */
+ } else if (line.startsWith("router-signature")) {
+ String[] routerLineParts = routerLine.split(" ");
+ scrubbedDesc = "router " + routerLineParts[1] + " "
+ + scrubbedAddress + " " + routerLineParts[3] + " "
+ + routerLineParts[4] + " " + routerLineParts[5] + "\n";
+ if (scrubbedOrAddresses != null) {
+ for (String scrubbedOrAddress : scrubbedOrAddresses) {
+ scrubbedDesc = scrubbedDesc += "or-address "
+ + scrubbedOrAddress + "\n";
+ }
+ }
+ scrubbedDesc += scrubbed.toString();
+ break;
+
+ /* Replace extra-info digest with the hashed digest of the
+ * non-scrubbed descriptor. */
+ } else if (line.startsWith("opt extra-info-digest ") ||
+ line.startsWith("extra-info-digest ")) {
+ String extraInfoDescriptorIdentifier = line.substring(
+ line.indexOf("extra-info-digest ")
+ + "extra-info-digest ".length());
+ String hashedExtraInfoDescriptorIdentifier =
+ DigestUtils.shaHex(Hex.decodeHex(
+ extraInfoDescriptorIdentifier.toCharArray())).toUpperCase();
+ scrubbed.append((line.startsWith("opt ") ? "opt " : "")
+ + "extra-info-digest " + hashedExtraInfoDescriptorIdentifier
+ + "\n");
+
+ /* Possibly sanitize reject lines if they contain the bridge's own
+ * IP address. */
+ } else if (line.startsWith("reject ")) {
+ if (address != null && line.startsWith("reject " + address)) {
+ scrubbed.append("reject " + scrubbedAddress
+ + line.substring("reject ".length() + address.length())
+ + "\n");
+ } else {
+ scrubbed.append(line + "\n");
+ }
+
+ /* Write the following lines unmodified to the sanitized
+ * descriptor. */
+ } else if (line.startsWith("accept ")
+ || line.startsWith("platform ")
+ || line.startsWith("opt protocols ")
+ || line.startsWith("protocols ")
+ || line.startsWith("uptime ")
+ || line.startsWith("bandwidth ")
+ || line.startsWith("opt hibernating ")
+ || line.startsWith("hibernating ")
+ || line.equals("opt hidden-service-dir")
+ || line.equals("hidden-service-dir")
+ || line.equals("opt caches-extra-info")
+ || line.equals("caches-extra-info")
+ || line.equals("opt allow-single-hop-exits")
+ || line.equals("allow-single-hop-exits")) {
+ scrubbed.append(line + "\n");
+
+ /* Replace node fingerprints in the family line with their hashes
+ * and leave nicknames unchanged. */
+ } else if (line.startsWith("family ")) {
+ StringBuilder familyLine = new StringBuilder("family");
+ for (String s : line.substring(7).split(" ")) {
+ if (s.startsWith("$")) {
+ familyLine.append(" $" + DigestUtils.shaHex(Hex.decodeHex(
+ s.substring(1).toCharArray())).toUpperCase());
+ } else {
+ familyLine.append(" " + s);
+ }
+ }
+ scrubbed.append(familyLine.toString() + "\n");
+
+ /* Skip the purpose line that the bridge authority adds to its
+ * cached-descriptors file. */
+ } else if (line.startsWith("@purpose ")) {
+ continue;
+
+ /* Skip all crypto parts that might leak the bridge's identity
+ * fingerprint. */
+ } else if (line.startsWith("-----BEGIN ")
+ || line.equals("onion-key") || line.equals("signing-key")) {
+ skipCrypto = true;
+
+ /* Stop skipping lines when the crypto parts are over. */
+ } else if (line.startsWith("-----END ")) {
+ skipCrypto = false;
+
+ /* If we encounter an unrecognized line, stop parsing and print
+ * out a warning. We might have overlooked sensitive information
+ * that we need to remove or replace for the sanitized descriptor
+ * version. */
+ } else {
+ this.logger.fine("Unrecognized line '" + line + "'. Skipping.");
+ return;
+ }
+ }
+ br.close();
+ } catch (Exception e) {
+ this.logger.log(Level.WARNING, "Could not parse server "
+ + "descriptor.", e);
+ return;
+ }
+
+ /* Determine filename of sanitized server descriptor. */
+ String descriptorDigest = null;
+ try {
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "router ";
+ String sigToken = "\nrouter-signature\n";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ if (start >= 0 && sig >= 0 && sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ descriptorDigest = DigestUtils.shaHex(DigestUtils.sha(forDigest));
+ }
+ } catch (UnsupportedEncodingException e) {
+ /* Handle below. */
+ }
+ if (descriptorDigest == null) {
+ this.logger.log(Level.WARNING, "Could not calculate server "
+ + "descriptor digest.");
+ return;
+ }
+ String dyear = published.substring(0, 4);
+ String dmonth = published.substring(5, 7);
+ File newFile = new File(
+ this.sanitizedBridgesDirectory.getAbsolutePath() + "/"
+ + dyear + "/" + dmonth + "/server-descriptors/"
+ + "/" + descriptorDigest.charAt(0) + "/"
+ + descriptorDigest.charAt(1) + "/"
+ + descriptorDigest);
+
+ /* Write sanitized server descriptor to disk, including all its parent
+ * directories. */
+ try {
+ newFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(newFile));
+ bw.write("@type bridge-server-descriptor 1.0\n");
+ bw.write(scrubbedDesc);
+ bw.write("router-digest " + descriptorDigest.toUpperCase() + "\n");
+ bw.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not write sanitized server "
+ + "descriptor to disk.", e);
+ return;
+ }
+ }
+
+ /**
+ * Sanitizes an extra-info descriptor and writes it to disk.
+ */
+ public void sanitizeAndStoreExtraInfoDescriptor(byte[] data) {
+
+ /* Parse descriptor to generate a sanitized version. */
+ String scrubbedDesc = null, published = null;
+ try {
+ BufferedReader br = new BufferedReader(new StringReader(new String(
+ data, "US-ASCII")));
+ String line = null;
+ StringBuilder scrubbed = null;
+ String hashedBridgeIdentity = null;
+ while ((line = br.readLine()) != null) {
+
+ /* Parse bridge identity from extra-info line and replace it with
+ * its hash in the sanitized descriptor. */
+ String[] parts = line.split(" ");
+ if (line.startsWith("extra-info ")) {
+ hashedBridgeIdentity = DigestUtils.shaHex(Hex.decodeHex(
+ parts[2].toCharArray())).toLowerCase();
+ scrubbed = new StringBuilder("extra-info " + parts[1] + " "
+ + hashedBridgeIdentity.toUpperCase() + "\n");
+
+ /* Parse the publication time to determine the file name. */
+ } else if (line.startsWith("published ")) {
+ scrubbed.append(line + "\n");
+ published = line.substring("published ".length());
+
+ /* Remove everything from transport lines except the transport
+ * name. */
+ } else if (line.startsWith("transport ")) {
+ if (parts.length < 3) {
+ this.logger.fine("Illegal line in extra-info descriptor: '"
+ + line + "'. Skipping descriptor.");
+ return;
+ }
+ scrubbed.append("transport " + parts[1] + "\n");
+
+ /* Skip transport-info lines entirely. */
+ } else if (line.startsWith("transport-info ")) {
+
+ /* Write the following lines unmodified to the sanitized
+ * descriptor. */
+ } else if (line.startsWith("write-history ")
+ || line.startsWith("read-history ")
+ || line.startsWith("geoip-start-time ")
+ || line.startsWith("geoip-client-origins ")
+ || line.startsWith("geoip-db-digest ")
+ || line.startsWith("conn-bi-direct ")
+ || line.startsWith("bridge-")
+ || line.startsWith("dirreq-")
+ || line.startsWith("cell-")
+ || line.startsWith("entry-")
+ || line.startsWith("exit-")) {
+ scrubbed.append(line + "\n");
+
+ /* When we reach the signature, we're done. Write the sanitized
+ * descriptor to disk below. */
+ } else if (line.startsWith("router-signature")) {
+ scrubbedDesc = scrubbed.toString();
+ break;
+
+ /* If we encounter an unrecognized line, stop parsing and print
+ * out a warning. We might have overlooked sensitive information
+ * that we need to remove or replace for the sanitized descriptor
+ * version. */
+ } else {
+ this.logger.fine("Unrecognized line '" + line + "'. Skipping.");
+ return;
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not parse extra-info "
+ + "descriptor.", e);
+ return;
+ } catch (DecoderException e) {
+ this.logger.log(Level.WARNING, "Could not parse extra-info "
+ + "descriptor.", e);
+ return;
+ }
+
+ /* Determine filename of sanitized extra-info descriptor. */
+ String descriptorDigest = null;
+ try {
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ if (start >= 0 && sig >= 0 && sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ descriptorDigest = DigestUtils.shaHex(DigestUtils.sha(forDigest));
+ }
+ } catch (UnsupportedEncodingException e) {
+ /* Handle below. */
+ }
+ if (descriptorDigest == null) {
+ this.logger.log(Level.WARNING, "Could not calculate extra-info "
+ + "descriptor digest.");
+ return;
+ }
+ String dyear = published.substring(0, 4);
+ String dmonth = published.substring(5, 7);
+ File newFile = new File(
+ this.sanitizedBridgesDirectory.getAbsolutePath() + "/"
+ + dyear + "/" + dmonth + "/extra-infos/"
+ + descriptorDigest.charAt(0) + "/"
+ + descriptorDigest.charAt(1) + "/"
+ + descriptorDigest);
+
+ /* Write sanitized extra-info descriptor to disk, including all its
+ * parent directories. */
+ try {
+ newFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(newFile));
+ bw.write("@type bridge-extra-info 1.1\n");
+ bw.write(scrubbedDesc);
+ bw.write("router-digest " + descriptorDigest.toUpperCase() + "\n");
+ bw.close();
+ } catch (Exception e) {
+ this.logger.log(Level.WARNING, "Could not write sanitized "
+ + "extra-info descriptor to disk.", e);
+ }
+ }
+
+ /**
+ * Rewrite all network statuses that might contain references to server
+ * descriptors we added or updated in this execution. This applies to
+ * all statuses that have been published up to 24 hours after any added
+ * or updated server descriptor.
+ */
+ public void finishWriting() {
+
+ /* Delete secrets that we don't need anymore. */
+ if (!this.secretsForHashingIPAddresses.isEmpty() &&
+ this.secretsForHashingIPAddresses.firstKey().compareTo(
+ this.bridgeSanitizingCutOffTimestamp) < 0) {
+ try {
+ int kept = 0, deleted = 0;
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.bridgeIpSecretsFile));
+ for (Map.Entry<String, byte[]> e :
+ this.secretsForHashingIPAddresses.entrySet()) {
+ if (e.getKey().compareTo(
+ this.bridgeSanitizingCutOffTimestamp) < 0) {
+ deleted++;
+ } else {
+ bw.write(e.getKey() + "," + Hex.encodeHexString(e.getValue())
+ + "\n");
+ kept++;
+ }
+ }
+ bw.close();
+ this.logger.info("Deleted " + deleted + " secrets that we don't "
+ + "need anymore and kept " + kept + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not store reduced set of "
+ + "secrets to disk! This is a bad sign, better check what's "
+ + "going on!", e);
+ }
+ }
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
new file mode 100644
index 0000000..470f6ab
--- /dev/null
+++ b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -0,0 +1,174 @@
+/* Copyright 2011--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.bridgepools;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.Stack;
+import java.util.TimeZone;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+
+public class BridgePoolAssignmentsProcessor {
+
+ public BridgePoolAssignmentsProcessor(File assignmentsDirectory,
+ File sanitizedAssignmentsDirectory) {
+
+ Logger logger =
+ Logger.getLogger(BridgePoolAssignmentsProcessor.class.getName());
+ if (assignmentsDirectory == null ||
+ sanitizedAssignmentsDirectory == null) {
+ IllegalArgumentException e = new IllegalArgumentException("Neither "
+ + "assignmentsDirectory nor sanitizedAssignmentsDirectory may "
+ + "be null!");
+ throw e;
+ }
+
+ List<File> assignmentFiles = new ArrayList<File>();
+ Stack<File> files = new Stack<File>();
+ files.add(assignmentsDirectory);
+ while (!files.isEmpty()) {
+ File file = files.pop();
+ if (file.isDirectory()) {
+ files.addAll(Arrays.asList(file.listFiles()));
+ } else if (!file.getName().endsWith(".gz")) {
+ assignmentFiles.add(file);
+ }
+ }
+
+ SimpleDateFormat assignmentFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ assignmentFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ SimpleDateFormat filenameFormat =
+ new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+ filenameFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ for (File assignmentFile : assignmentFiles) {
+ logger.info("Processing bridge pool assignment file '"
+ + assignmentFile.getAbsolutePath() + "'...");
+ try {
+ BufferedReader br = null;
+ if (assignmentFile.getName().endsWith(".gz")) {
+ br = new BufferedReader(new InputStreamReader(
+ new GzipCompressorInputStream(new FileInputStream(
+ assignmentFile))));
+ } else {
+ br = new BufferedReader(new FileReader(assignmentFile));
+ }
+ String line, bridgePoolAssignmentLine = null;
+ SortedSet<String> sanitizedAssignments = new TreeSet<String>();
+ boolean wroteLastLine = false, skipBefore20120504125947 = true;
+ while ((line = br.readLine()) != null || !wroteLastLine) {
+ if (line != null && line.startsWith("bridge-pool-assignment ")) {
+ String[] parts = line.split(" ");
+ if (parts.length != 3) {
+ continue;
+ }
+ /* TODO Take out this temporary hack to ignore all assignments
+ * coming from ponticum when byblos was still the official
+ * BridgeDB host. */
+ if (line.compareTo(
+ "bridge-pool-assignment 2012-05-04 12:59:47") >= 0) {
+ skipBefore20120504125947 = false;
+ }
+ }
+ if (skipBefore20120504125947) {
+ if (line == null) {
+ break;
+ } else {
+ continue;
+ }
+ }
+ if (line == null ||
+ line.startsWith("bridge-pool-assignment ")) {
+ if (bridgePoolAssignmentLine != null) {
+ try {
+ long bridgePoolAssignmentTime = assignmentFormat.parse(
+ bridgePoolAssignmentLine.substring(
+ "bridge-pool-assignment ".length())).getTime();
+ File sanitizedAssignmentsFile = new File(
+ sanitizedAssignmentsDirectory, filenameFormat.format(
+ bridgePoolAssignmentTime));
+ if (!sanitizedAssignmentsFile.exists()) {
+ sanitizedAssignmentsFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ sanitizedAssignmentsFile));
+ bw.write("@type bridge-pool-assignment 1.0\n");
+ bw.write(bridgePoolAssignmentLine + "\n");
+ for (String assignmentLine : sanitizedAssignments) {
+ bw.write(assignmentLine + "\n");
+ }
+ bw.close();
+ }
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not write sanitized "
+ + "bridge pool assignment file for line '"
+ + bridgePoolAssignmentLine + "' to disk. Skipping "
+ + "bridge pool assignment file '"
+ + assignmentFile.getAbsolutePath() + "'.", e);
+ break;
+ } catch (ParseException e) {
+ logger.log(Level.WARNING, "Could not write sanitized "
+ + "bridge pool assignment file for line '"
+ + bridgePoolAssignmentLine + "' to disk. Skipping "
+ + "bridge pool assignment file '"
+ + assignmentFile.getAbsolutePath() + "'.", e);
+ break;
+ }
+ sanitizedAssignments.clear();
+ }
+ if (line == null) {
+ wroteLastLine = true;
+ } else {
+ bridgePoolAssignmentLine = line;
+ }
+ } else {
+ String[] parts = line.split(" ");
+ if (parts.length < 2 || parts[0].length() < 40) {
+ logger.warning("Unrecognized line '" + line
+ + "'. Aborting.");
+ break;
+ }
+ String hashedFingerprint = null;
+ try {
+ hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex(
+ line.split(" ")[0].toCharArray())).toLowerCase();
+ } catch (DecoderException e) {
+ logger.warning("Unable to decode hex fingerprint in line '"
+ + line + "'. Aborting.");
+ break;
+ }
+ String assignmentDetails = line.substring(40);
+ sanitizedAssignments.add(hashedFingerprint
+ + assignmentDetails);
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not read bridge pool assignment "
+ + "file '" + assignmentFile.getAbsolutePath()
+ + "'. Skipping.", e);
+ }
+ }
+
+ logger.info("Finished processing bridge pool assignment file(s).");
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
new file mode 100644
index 0000000..64f6a3b
--- /dev/null
+++ b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
@@ -0,0 +1,100 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.exitlists;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.SortedSet;
+import java.util.Stack;
+import java.util.TimeZone;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class ExitListDownloader {
+ public ExitListDownloader() {
+ Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
+ try {
+ logger.fine("Downloading exit list...");
+ String exitAddressesUrl =
+ "http://exitlist.torproject.org/exit-addresses";;
+ URL u = new URL(exitAddressesUrl);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ int response = huc.getResponseCode();
+ if (response != 200) {
+ logger.warning("Could not download exit list. Response code " +
+ response);
+ return;
+ }
+ BufferedInputStream in = new BufferedInputStream(
+ huc.getInputStream());
+ SimpleDateFormat printFormat =
+ new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ Date downloadedDate = new Date();
+ File exitListFile = new File("exitlist/" + printFormat.format(
+ downloadedDate));
+ exitListFile.getParentFile().mkdirs();
+ SimpleDateFormat dateTimeFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ exitListFile));
+ bw.write("@type tordnsel 1.0\n");
+ bw.write("Downloaded " + dateTimeFormat.format(downloadedDate)
+ + "\n");
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = in.read(data, 0, 1024)) >= 0) {
+ bw.write(new String(data, 0, len));
+ }
+ in.close();
+ bw.close();
+ logger.fine("Finished downloading exit list.");
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed downloading exit list", e);
+ return;
+ }
+
+ /* Write stats. */
+ StringBuilder dumpStats = new StringBuilder("Finished downloading "
+ + "exit list.\nLast three exit lists are:");
+ Stack<File> filesInInputDir = new Stack<File>();
+ filesInInputDir.add(new File("exitlist"));
+ SortedSet<File> lastThreeExitLists = new TreeSet<File>();
+ while (!filesInInputDir.isEmpty()) {
+ File pop = filesInInputDir.pop();
+ if (pop.isDirectory()) {
+ SortedSet<File> lastThreeElements = new TreeSet<File>();
+ for (File f : pop.listFiles()) {
+ lastThreeElements.add(f);
+ }
+ while (lastThreeElements.size() > 3) {
+ lastThreeElements.remove(lastThreeElements.first());
+ }
+ for (File f : lastThreeElements) {
+ filesInInputDir.add(f);
+ }
+ } else {
+ lastThreeExitLists.add(pop);
+ while (lastThreeExitLists.size() > 3) {
+ lastThreeExitLists.remove(lastThreeExitLists.first());
+ }
+ }
+ }
+ for (File f : lastThreeExitLists) {
+ dumpStats.append("\n" + f.getName());
+ }
+ logger.info(dumpStats.toString());
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/main/Configuration.java b/src/org/torproject/ernie/db/main/Configuration.java
new file mode 100644
index 0000000..adf22cc
--- /dev/null
+++ b/src/org/torproject/ernie/db/main/Configuration.java
@@ -0,0 +1,359 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.main;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Initialize configuration with hard-coded defaults, overwrite with
+ * configuration in config file, if exists, and answer Main.java about our
+ * configuration.
+ */
+public class Configuration {
+ private boolean writeDirectoryArchives = false;
+ private String directoryArchivesOutputDirectory = "directory-archive/";
+ private boolean importCachedRelayDescriptors = false;
+ private List<String> cachedRelayDescriptorsDirectory =
+ new ArrayList<String>(Arrays.asList("cacheddesc/".split(",")));
+ private boolean importDirectoryArchives = false;
+ private String directoryArchivesDirectory = "archives/";
+ private boolean keepDirectoryArchiveImportHistory = false;
+ private boolean writeSanitizedBridges = false;
+ private boolean replaceIPAddressesWithHashes = false;
+ private long limitBridgeDescriptorMappings = -1L;
+ private String sanitizedBridgesWriteDirectory = "sanitized-bridges/";
+ private boolean importBridgeSnapshots = false;
+ private String bridgeSnapshotsDirectory = "bridge-directories/";
+ private boolean downloadRelayDescriptors = false;
+ private List<String> downloadFromDirectoryAuthorities = Arrays.asList((
+ "86.59.21.38,76.73.17.194:9030,213.115.239.118:443,"
+ + "193.23.244.244,208.83.223.34:443,128.31.0.34:9131,"
+ + "194.109.206.212,212.112.245.170").split(","));
+ private boolean downloadCurrentConsensus = true;
+ private boolean downloadCurrentVotes = true;
+ private boolean downloadMissingServerDescriptors = true;
+ private boolean downloadMissingExtraInfoDescriptors = true;
+ private boolean downloadAllServerDescriptors = false;
+ private boolean downloadAllExtraInfoDescriptors = false;
+ private boolean compressRelayDescriptorDownloads;
+ private boolean downloadExitList = false;
+ private boolean processBridgePoolAssignments = false;
+ private String assignmentsDirectory = "assignments/";
+ private String sanitizedAssignmentsDirectory = "sanitized-assignments/";
+ private boolean processTorperfFiles = false;
+ private String torperfOutputDirectory = "torperf/";
+ private SortedMap<String, String> torperfSources = null;
+ private List<String> torperfFiles = null;
+ private boolean provideFilesViaRsync = false;
+ private String rsyncDirectory = "rsync";
+ public Configuration() {
+
+ /* Initialize logger. */
+ Logger logger = Logger.getLogger(Configuration.class.getName());
+
+ /* Read config file, if present. */
+ File configFile = new File("config");
+ if (!configFile.exists()) {
+ logger.warning("Could not find config file. In the default "
+ + "configuration, we are not configured to read data from any "
+ + "data source or write data to any data sink. You need to "
+ + "create a config file (" + configFile.getAbsolutePath()
+ + ") and provide at least one data source and one data sink. "
+ + "Refer to the manual for more information.");
+ return;
+ }
+ String line = null;
+ boolean containsCachedRelayDescriptorsDirectory = false;
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(configFile));
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("#") || line.length() < 1) {
+ continue;
+ } else if (line.startsWith("WriteDirectoryArchives")) {
+ this.writeDirectoryArchives = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DirectoryArchivesOutputDirectory")) {
+ this.directoryArchivesOutputDirectory = line.split(" ")[1];
+ } else if (line.startsWith("ImportCachedRelayDescriptors")) {
+ this.importCachedRelayDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("CachedRelayDescriptorsDirectory")) {
+ if (!containsCachedRelayDescriptorsDirectory) {
+ this.cachedRelayDescriptorsDirectory.clear();
+ containsCachedRelayDescriptorsDirectory = true;
+ }
+ this.cachedRelayDescriptorsDirectory.add(line.split(" ")[1]);
+ } else if (line.startsWith("ImportDirectoryArchives")) {
+ this.importDirectoryArchives = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DirectoryArchivesDirectory")) {
+ this.directoryArchivesDirectory = line.split(" ")[1];
+ } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
+ this.keepDirectoryArchiveImportHistory = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("WriteSanitizedBridges")) {
+ this.writeSanitizedBridges = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("ReplaceIPAddressesWithHashes")) {
+ this.replaceIPAddressesWithHashes = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("LimitBridgeDescriptorMappings")) {
+ this.limitBridgeDescriptorMappings = Long.parseLong(
+ line.split(" ")[1]);
+ } else if (line.startsWith("SanitizedBridgesWriteDirectory")) {
+ this.sanitizedBridgesWriteDirectory = line.split(" ")[1];
+ } else if (line.startsWith("ImportBridgeSnapshots")) {
+ this.importBridgeSnapshots = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("BridgeSnapshotsDirectory")) {
+ this.bridgeSnapshotsDirectory = line.split(" ")[1];
+ } else if (line.startsWith("DownloadRelayDescriptors")) {
+ this.downloadRelayDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadFromDirectoryAuthorities")) {
+ this.downloadFromDirectoryAuthorities = new ArrayList<String>();
+ for (String dir : line.split(" ")[1].split(",")) {
+ // test if IP:port pair has correct format
+ if (dir.length() < 1) {
+ logger.severe("Configuration file contains directory "
+ + "authority IP:port of length 0 in line '" + line
+ + "'! Exiting!");
+ System.exit(1);
+ }
+ new URL("http://"; + dir + "/");
+ this.downloadFromDirectoryAuthorities.add(dir);
+ }
+ } else if (line.startsWith("DownloadCurrentConsensus")) {
+ this.downloadCurrentConsensus = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadCurrentVotes")) {
+ this.downloadCurrentVotes = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadMissingServerDescriptors")) {
+ this.downloadMissingServerDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith(
+ "DownloadMissingExtraInfoDescriptors")) {
+ this.downloadMissingExtraInfoDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadAllServerDescriptors")) {
+ this.downloadAllServerDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadAllExtraInfoDescriptors")) {
+ this.downloadAllExtraInfoDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("CompressRelayDescriptorDownloads")) {
+ this.compressRelayDescriptorDownloads = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadExitList")) {
+ this.downloadExitList = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("ProcessBridgePoolAssignments")) {
+ this.processBridgePoolAssignments = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("AssignmentsDirectory")) {
+ this.assignmentsDirectory = line.split(" ")[1];
+ } else if (line.startsWith("SanitizedAssignmentsDirectory")) {
+ this.sanitizedAssignmentsDirectory = line.split(" ")[1];
+ } else if (line.startsWith("ProcessTorperfFiles")) {
+ this.processTorperfFiles = Integer.parseInt(line.split(" ")[1])
+ != 0;
+ } else if (line.startsWith("TorperfOutputDirectory")) {
+ } else if (line.startsWith("TorperfSource")) {
+ if (this.torperfSources == null) {
+ this.torperfSources = new TreeMap<String, String>();
+ }
+ String[] parts = line.split(" ");
+ String sourceName = parts[1];
+ String baseUrl = parts[2];
+ this.torperfSources.put(sourceName, baseUrl);
+ } else if (line.startsWith("TorperfFiles")) {
+ if (this.torperfFiles == null) {
+ this.torperfFiles = new ArrayList<String>();
+ }
+ String[] parts = line.split(" ");
+ if (parts.length != 5) {
+ logger.severe("Configuration file contains TorperfFiles "
+ + "option with wrong number of values in line '" + line
+ + "'! Exiting!");
+ System.exit(1);
+ }
+ this.torperfFiles.add(line);
+ } else if (line.startsWith("ProvideFilesViaRsync")) {
+ this.provideFilesViaRsync = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("RsyncDirectory")) {
+ this.rsyncDirectory = line.split(" ")[1];
+ } else {
+ logger.severe("Configuration file contains unrecognized "
+ + "configuration key in line '" + line + "'! Exiting!");
+ System.exit(1);
+ }
+ }
+ br.close();
+ } catch (ArrayIndexOutOfBoundsException e) {
+ logger.severe("Configuration file contains configuration key "
+ + "without value in line '" + line + "'. Exiting!");
+ System.exit(1);
+ } catch (MalformedURLException e) {
+ logger.severe("Configuration file contains illegal URL or IP:port "
+ + "pair in line '" + line + "'. Exiting!");
+ System.exit(1);
+ } catch (NumberFormatException e) {
+ logger.severe("Configuration file contains illegal value in line '"
+ + line + "' with legal values being 0 or 1. Exiting!");
+ System.exit(1);
+ } catch (IOException e) {
+ logger.log(Level.SEVERE, "Unknown problem while reading config "
+ + "file! Exiting!", e);
+ System.exit(1);
+ }
+
+ /** Make some checks if configuration is valid. */
+ if (!this.importCachedRelayDescriptors &&
+ !this.importDirectoryArchives && !this.downloadRelayDescriptors &&
+ !this.importBridgeSnapshots &&
+ !this.downloadExitList && !this.processBridgePoolAssignments &&
+ !this.writeDirectoryArchives && !this.writeSanitizedBridges &&
+ !this.processTorperfFiles) {
+ logger.warning("We have not been configured to read data from any "
+ + "data source or write data to any data sink. You need to "
+ + "edit your config file (" + configFile.getAbsolutePath()
+ + ") and provide at least one data source and one data sink. "
+ + "Refer to the manual for more information.");
+ }
+ if ((this.importCachedRelayDescriptors ||
+ this.importDirectoryArchives || this.downloadRelayDescriptors) &&
+ !this.writeDirectoryArchives) {
+ logger.warning("We are configured to import/download relay "
+ + "descriptors, but we don't have a single data sink to write "
+ + "relay descriptors to.");
+ }
+ if (!(this.importCachedRelayDescriptors ||
+ this.importDirectoryArchives || this.downloadRelayDescriptors) &&
+ this.writeDirectoryArchives) {
+ logger.warning("We are configured to write relay descriptor to at "
+ + "least one data sink, but we don't have a single data source "
+ + "containing relay descriptors.");
+ }
+ if (this.importBridgeSnapshots && !this.writeSanitizedBridges) {
+ logger.warning("We are configured to import/download bridge "
+ + "descriptors, but we don't have a single data sink to write "
+ + "bridge descriptors to.");
+ }
+ if (!this.importBridgeSnapshots && this.writeSanitizedBridges) {
+ logger.warning("We are configured to write bridge descriptor to at "
+ + "least one data sink, but we don't have a single data source "
+ + "containing bridge descriptors.");
+ }
+ }
+ public boolean getWriteDirectoryArchives() {
+ return this.writeDirectoryArchives;
+ }
+ public String getDirectoryArchivesOutputDirectory() {
+ return this.directoryArchivesOutputDirectory;
+ }
+ public boolean getImportCachedRelayDescriptors() {
+ return this.importCachedRelayDescriptors;
+ }
+ public List<String> getCachedRelayDescriptorDirectory() {
+ return this.cachedRelayDescriptorsDirectory;
+ }
+ public boolean getImportDirectoryArchives() {
+ return this.importDirectoryArchives;
+ }
+ public String getDirectoryArchivesDirectory() {
+ return this.directoryArchivesDirectory;
+ }
+ public boolean getKeepDirectoryArchiveImportHistory() {
+ return this.keepDirectoryArchiveImportHistory;
+ }
+ public boolean getWriteSanitizedBridges() {
+ return this.writeSanitizedBridges;
+ }
+ public boolean getReplaceIPAddressesWithHashes() {
+ return this.replaceIPAddressesWithHashes;
+ }
+ public long getLimitBridgeDescriptorMappings() {
+ return this.limitBridgeDescriptorMappings;
+ }
+ public String getSanitizedBridgesWriteDirectory() {
+ return this.sanitizedBridgesWriteDirectory;
+ }
+ public boolean getImportBridgeSnapshots() {
+ return this.importBridgeSnapshots;
+ }
+ public String getBridgeSnapshotsDirectory() {
+ return this.bridgeSnapshotsDirectory;
+ }
+ public boolean getDownloadRelayDescriptors() {
+ return this.downloadRelayDescriptors;
+ }
+ public List<String> getDownloadFromDirectoryAuthorities() {
+ return this.downloadFromDirectoryAuthorities;
+ }
+ public boolean getDownloadCurrentConsensus() {
+ return this.downloadCurrentConsensus;
+ }
+ public boolean getDownloadCurrentVotes() {
+ return this.downloadCurrentVotes;
+ }
+ public boolean getDownloadMissingServerDescriptors() {
+ return this.downloadMissingServerDescriptors;
+ }
+ public boolean getDownloadMissingExtraInfoDescriptors() {
+ return this.downloadMissingExtraInfoDescriptors;
+ }
+ public boolean getDownloadAllServerDescriptors() {
+ return this.downloadAllServerDescriptors;
+ }
+ public boolean getDownloadAllExtraInfoDescriptors() {
+ return this.downloadAllExtraInfoDescriptors;
+ }
+ public boolean getCompressRelayDescriptorDownloads() {
+ return this.compressRelayDescriptorDownloads;
+ }
+ public boolean getDownloadExitList() {
+ return this.downloadExitList;
+ }
+ public boolean getProcessBridgePoolAssignments() {
+ return processBridgePoolAssignments;
+ }
+ public String getAssignmentsDirectory() {
+ return assignmentsDirectory;
+ }
+ public String getSanitizedAssignmentsDirectory() {
+ return sanitizedAssignmentsDirectory;
+ }
+ public boolean getProcessTorperfFiles() {
+ return this.processTorperfFiles;
+ }
+ public String getTorperfOutputDirectory() {
+ return this.torperfOutputDirectory;
+ }
+ public SortedMap<String, String> getTorperfSources() {
+ return this.torperfSources;
+ }
+ public List<String> getTorperfFiles() {
+ return this.torperfFiles;
+ }
+ public boolean getProvideFilesViaRsync() {
+ return this.provideFilesViaRsync;
+ }
+ public String getRsyncDirectory() {
+ return this.rsyncDirectory;
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/main/LockFile.java b/src/org/torproject/ernie/db/main/LockFile.java
new file mode 100644
index 0000000..68375ec
--- /dev/null
+++ b/src/org/torproject/ernie/db/main/LockFile.java
@@ -0,0 +1,52 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.main;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.logging.Logger;
+
+public class LockFile {
+
+ private File lockFile;
+ private Logger logger;
+
+ public LockFile() {
+ this.lockFile = new File("lock");
+ this.logger = Logger.getLogger(LockFile.class.getName());
+ }
+
+ public boolean acquireLock() {
+ this.logger.fine("Trying to acquire lock...");
+ try {
+ if (this.lockFile.exists()) {
+ BufferedReader br = new BufferedReader(new FileReader("lock"));
+ long runStarted = Long.parseLong(br.readLine());
+ br.close();
+ if (System.currentTimeMillis() - runStarted < 55L * 60L * 1000L) {
+ return false;
+ }
+ }
+ BufferedWriter bw = new BufferedWriter(new FileWriter("lock"));
+ bw.append("" + System.currentTimeMillis() + "\n");
+ bw.close();
+ this.logger.fine("Acquired lock.");
+ return true;
+ } catch (IOException e) {
+ this.logger.warning("Caught exception while trying to acquire "
+ + "lock!");
+ return false;
+ }
+ }
+
+ public void releaseLock() {
+ this.logger.fine("Releasing lock...");
+ this.lockFile.delete();
+ this.logger.fine("Released lock.");
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/main/LoggingConfiguration.java b/src/org/torproject/ernie/db/main/LoggingConfiguration.java
new file mode 100644
index 0000000..b0ddeaa
--- /dev/null
+++ b/src/org/torproject/ernie/db/main/LoggingConfiguration.java
@@ -0,0 +1,93 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.main;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+import java.util.logging.ConsoleHandler;
+import java.util.logging.FileHandler;
+import java.util.logging.Formatter;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+/**
+ * Initialize logging configuration.
+ *
+ * Log levels used by ERNIE:
+ *
+ * - SEVERE: An event made it impossible to continue program execution.
+ * - WARNING: A potential problem occurred that requires the operator to
+ * look after the otherwise unattended setup
+ * - INFO: Messages on INFO level are meant to help the operator in making
+ * sure that operation works as expected.
+ * - FINE: Debug messages that are used to identify problems and which are
+ * turned on by default.
+ * - FINER: More detailed debug messages to investigate problems in more
+ * detail. Not turned on by default. Increase log file limit when using
+ * FINER.
+ * - FINEST: Most detailed debug messages. Not used.
+ */
+public class LoggingConfiguration {
+ public LoggingConfiguration() {
+
+ /* Remove default console handler. */
+ for (Handler h : Logger.getLogger("").getHandlers()) {
+ Logger.getLogger("").removeHandler(h);
+ }
+
+ /* Disable logging of internal Sun classes. */
+ Logger.getLogger("sun").setLevel(Level.OFF);
+
+ /* Set minimum log level we care about from INFO to FINER. */
+ Logger.getLogger("").setLevel(Level.FINER);
+
+ /* Create log handler that writes messages on WARNING or higher to the
+ * console. */
+ final SimpleDateFormat dateTimeFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ Formatter cf = new Formatter() {
+ public String format(LogRecord record) {
+ return dateTimeFormat.format(new Date(record.getMillis())) + " "
+ + record.getMessage() + "\n";
+ }
+ };
+ Handler ch = new ConsoleHandler();
+ ch.setFormatter(cf);
+ ch.setLevel(Level.WARNING);
+ Logger.getLogger("").addHandler(ch);
+
+ /* Initialize own logger for this class. */
+ Logger logger = Logger.getLogger(
+ LoggingConfiguration.class.getName());
+
+ /* Create log handler that writes all messages on FINE or higher to a
+ * local file. */
+ Formatter ff = new Formatter() {
+ public String format(LogRecord record) {
+ return dateTimeFormat.format(new Date(record.getMillis())) + " "
+ + record.getLevel() + " " + record.getSourceClassName() + " "
+ + record.getSourceMethodName() + " " + record.getMessage()
+ + (record.getThrown() != null ? " " + record.getThrown() : "")
+ + "\n";
+ }
+ };
+ try {
+ FileHandler fh = new FileHandler("log", 5000000, 5, true);
+ fh.setFormatter(ff);
+ fh.setLevel(Level.FINE);
+ Logger.getLogger("").addHandler(fh);
+ } catch (SecurityException e) {
+ logger.log(Level.WARNING, "No permission to create log file. "
+ + "Logging to file is disabled.", e);
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not write to log file. Logging to "
+ + "file is disabled.", e);
+ }
+ }
+}
diff --git a/src/org/torproject/ernie/db/main/Main.java b/src/org/torproject/ernie/db/main/Main.java
new file mode 100644
index 0000000..e008eca
--- /dev/null
+++ b/src/org/torproject/ernie/db/main/Main.java
@@ -0,0 +1,172 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.main;
+
+import java.io.File;
+import java.util.List;
+import java.util.logging.Logger;
+
+import org.torproject.ernie.db.bridgedescs.BridgeDescriptorParser;
+import org.torproject.ernie.db.bridgedescs.BridgeSnapshotReader;
+import org.torproject.ernie.db.bridgedescs.SanitizedBridgesWriter;
+import org.torproject.ernie.db.bridgepools.BridgePoolAssignmentsProcessor;
+import org.torproject.ernie.db.exitlists.ExitListDownloader;
+import org.torproject.ernie.db.relaydescs.ArchiveReader;
+import org.torproject.ernie.db.relaydescs.ArchiveWriter;
+import org.torproject.ernie.db.relaydescs.CachedRelayDescriptorReader;
+import org.torproject.ernie.db.relaydescs.RelayDescriptorDownloader;
+import org.torproject.ernie.db.relaydescs.RelayDescriptorParser;
+import org.torproject.ernie.db.torperf.TorperfDownloader;
+
+/**
+ * Coordinate downloading and parsing of descriptors and extraction of
+ * statistically relevant data for later processing with R.
+ */
+public class Main {
+ public static void main(String[] args) {
+
+ /* Initialize logging configuration. */
+ new LoggingConfiguration();
+
+ Logger logger = Logger.getLogger(Main.class.getName());
+ logger.info("Starting ERNIE.");
+
+ // Initialize configuration
+ Configuration config = new Configuration();
+
+ // Use lock file to avoid overlapping runs
+ LockFile lf = new LockFile();
+ if (!lf.acquireLock()) {
+ logger.severe("Warning: ERNIE is already running or has not exited "
+ + "cleanly! Exiting!");
+ System.exit(1);
+ }
+
+ // Define stats directory for temporary files
+ File statsDirectory = new File("stats");
+
+ // Prepare writing relay descriptor archive to disk
+ ArchiveWriter aw = config.getWriteDirectoryArchives() ?
+ new ArchiveWriter(
+ new File(config.getDirectoryArchivesOutputDirectory())) : null;
+
+ // Prepare relay descriptor parser (only if we are writing stats or
+ // directory archives to disk)
+ RelayDescriptorParser rdp = aw != null ?
+ new RelayDescriptorParser(aw) : null;
+
+ // Import/download relay descriptors from the various sources
+ if (rdp != null) {
+ RelayDescriptorDownloader rdd = null;
+ if (config.getDownloadRelayDescriptors()) {
+ List<String> dirSources =
+ config.getDownloadFromDirectoryAuthorities();
+ rdd = new RelayDescriptorDownloader(rdp, dirSources,
+ config.getDownloadCurrentConsensus(),
+ config.getDownloadCurrentVotes(),
+ config.getDownloadMissingServerDescriptors(),
+ config.getDownloadMissingExtraInfoDescriptors(),
+ config.getDownloadAllServerDescriptors(),
+ config.getDownloadAllExtraInfoDescriptors(),
+ config.getCompressRelayDescriptorDownloads());
+ rdp.setRelayDescriptorDownloader(rdd);
+ }
+ if (config.getImportCachedRelayDescriptors()) {
+ new CachedRelayDescriptorReader(rdp,
+ config.getCachedRelayDescriptorDirectory(), statsDirectory);
+ if (aw != null) {
+ aw.intermediateStats("importing relay descriptors from local "
+ + "Tor data directories");
+ }
+ }
+ if (config.getImportDirectoryArchives()) {
+ new ArchiveReader(rdp,
+ new File(config.getDirectoryArchivesDirectory()),
+ statsDirectory,
+ config.getKeepDirectoryArchiveImportHistory());
+ if (aw != null) {
+ aw.intermediateStats("importing relay descriptors from local "
+ + "directory");
+ }
+ }
+ if (rdd != null) {
+ rdd.downloadDescriptors();
+ rdd.writeFile();
+ rdd = null;
+ if (aw != null) {
+ aw.intermediateStats("downloading relay descriptors from the "
+ + "directory authorities");
+ }
+ }
+ }
+
+ // Write output to disk that only depends on relay descriptors
+ if (aw != null) {
+ aw.dumpStats();
+ aw = null;
+ }
+
+ // Prepare sanitized bridge descriptor writer
+ SanitizedBridgesWriter sbw = config.getWriteSanitizedBridges() ?
+ new SanitizedBridgesWriter(
+ new File(config.getSanitizedBridgesWriteDirectory()),
+ statsDirectory, config.getReplaceIPAddressesWithHashes(),
+ config.getLimitBridgeDescriptorMappings()) : null;
+
+ // Prepare bridge descriptor parser
+ BridgeDescriptorParser bdp = config.getWriteSanitizedBridges()
+ ? new BridgeDescriptorParser(sbw) : null;
+
+ // Import bridge descriptors
+ if (bdp != null && config.getImportBridgeSnapshots()) {
+ new BridgeSnapshotReader(bdp,
+ new File(config.getBridgeSnapshotsDirectory()),
+ statsDirectory);
+ }
+
+ // Finish writing sanitized bridge descriptors to disk
+ if (sbw != null) {
+ sbw.finishWriting();
+ sbw = null;
+ }
+
+ // Download exit list and store it to disk
+ if (config.getDownloadExitList()) {
+ new ExitListDownloader();
+ }
+
+ // Process bridge pool assignments
+ if (config.getProcessBridgePoolAssignments()) {
+ new BridgePoolAssignmentsProcessor(
+ new File(config.getAssignmentsDirectory()),
+ new File(config.getSanitizedAssignmentsDirectory()));
+ }
+
+ // Process Torperf files
+ if (config.getProcessTorperfFiles()) {
+ new TorperfDownloader(new File(config.getTorperfOutputDirectory()),
+ config.getTorperfSources(), config.getTorperfFiles());
+ }
+
+ // Copy recently published files to a local directory that can then
+ // be served via rsync.
+ if (config.getProvideFilesViaRsync()) {
+ new RsyncDataProvider(
+ !config.getWriteDirectoryArchives() ? null :
+ new File(config.getDirectoryArchivesOutputDirectory()),
+ !config.getWriteSanitizedBridges() ? null :
+ new File(config.getSanitizedBridgesWriteDirectory()),
+ !config.getProcessBridgePoolAssignments() ? null :
+ new File(config.getSanitizedAssignmentsDirectory()),
+ config.getDownloadExitList(),
+ !config.getProcessTorperfFiles() ? null :
+ new File(config.getTorperfOutputDirectory()),
+ new File(config.getRsyncDirectory()));
+ }
+
+ // Remove lock file
+ lf.releaseLock();
+
+ logger.info("Terminating ERNIE.");
+ }
+}
diff --git a/src/org/torproject/ernie/db/main/RsyncDataProvider.java b/src/org/torproject/ernie/db/main/RsyncDataProvider.java
new file mode 100644
index 0000000..cd4a6f9
--- /dev/null
+++ b/src/org/torproject/ernie/db/main/RsyncDataProvider.java
@@ -0,0 +1,217 @@
+/* Copyright 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.main;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.Stack;
+import java.util.logging.Logger;
+
+/**
+ * Copy files published in the last 3 days to a local directory that can
+ * then be served via rsync.
+ */
+public class RsyncDataProvider {
+ public RsyncDataProvider(File directoryArchivesOutputDirectory,
+ File sanitizedBridgesWriteDirectory,
+ File sanitizedAssignmentsDirectory,
+ boolean downloadExitList,
+ File torperfOutputDirectory, File rsyncDirectory) {
+
+ /* Initialize logger. */
+ Logger logger = Logger.getLogger(RsyncDataProvider.class.getName());
+
+ /* Determine the cut-off time for files in rsync/. */
+ long cutOffMillis = System.currentTimeMillis()
+ - 3L * 24L * 60L * 60L * 1000L;
+
+ /* Create rsync/ directory if it doesn't exist. */
+ if (!rsyncDirectory.exists()) {
+ rsyncDirectory.mkdirs();
+ }
+
+ /* Make a list of all files in the rsync/ directory to delete those
+ * that we didn't copy in this run. */
+ Set<String> fileNamesInRsync = new HashSet<String>();
+ Stack<File> files = new Stack<File>();
+ files.add(rsyncDirectory);
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else {
+ fileNamesInRsync.add(pop.getName());
+ }
+ }
+ logger.info("Found " + fileNamesInRsync.size() + " files in "
+ + rsyncDirectory.getAbsolutePath() + " that we're either "
+ + "overwriting or deleting in this execution.");
+
+ /* Copy relay descriptors from the last 3 days. */
+ if (directoryArchivesOutputDirectory != null) {
+ files.add(directoryArchivesOutputDirectory);
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else if (pop.lastModified() >= cutOffMillis) {
+ String fileName = pop.getName();
+ if (pop.getAbsolutePath().contains("/consensus/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "relay-descriptors/consensuses/" + fileName));
+ } else if (pop.getAbsolutePath().contains("/vote/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "relay-descriptors/votes/" + fileName));
+ } else if (pop.getAbsolutePath().contains(
+ "/server-descriptor/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "relay-descriptors/server-descriptors/" + fileName));
+ } else if (pop.getAbsolutePath().contains("/extra-info/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "relay-descriptors/extra-infos/" + fileName));
+ } else {
+ continue;
+ }
+ fileNamesInRsync.remove(pop.getName());
+ }
+ }
+ }
+ logger.info("After copying relay descriptors, there are still "
+ + fileNamesInRsync.size() + " files left in "
+ + rsyncDirectory.getAbsolutePath() + ".");
+
+ /* Copy sanitized bridge descriptors from the last 3 days. */
+ if (sanitizedBridgesWriteDirectory != null) {
+ files.add(sanitizedBridgesWriteDirectory);
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else if (pop.lastModified() >= cutOffMillis) {
+ String fileName = pop.getName();
+ if (pop.getAbsolutePath().contains("/statuses/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "bridge-descriptors/statuses/" + fileName));
+ } else if (pop.getAbsolutePath().contains(
+ "/server-descriptors/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "bridge-descriptors/server-descriptors/" + fileName));
+ } else if (pop.getAbsolutePath().contains("/extra-infos/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "bridge-descriptors/extra-infos/" + fileName));
+ } else {
+ continue;
+ }
+ fileNamesInRsync.remove(pop.getName());
+ }
+ }
+ }
+ logger.info("After copying sanitized bridge descriptors, there are "
+ + "still " + fileNamesInRsync.size() + " files left in "
+ + rsyncDirectory.getAbsolutePath() + ".");
+
+ /* Copy sanitized bridge pool assignments from the last 3 days. */
+ if (sanitizedAssignmentsDirectory != null) {
+ files.add(sanitizedAssignmentsDirectory);
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else if (pop.lastModified() >= cutOffMillis) {
+ String fileName = pop.getName();
+ this.copyFile(pop, new File(rsyncDirectory,
+ "bridge-pool-assignments/" + fileName));
+ fileNamesInRsync.remove(pop.getName());
+ }
+ }
+ }
+ logger.info("After copying sanitized bridge pool assignments, there "
+ + "are still " + fileNamesInRsync.size() + " files left in "
+ + rsyncDirectory.getAbsolutePath() + ".");
+
+ /* Copy exit lists from the last 3 days. */
+ if (downloadExitList) {
+ files.add(new File("exitlist"));
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else if (pop.lastModified() >= cutOffMillis) {
+ String fileName = pop.getName();
+ this.copyFile(pop, new File(rsyncDirectory,
+ "exit-lists/" + fileName));
+ fileNamesInRsync.remove(pop.getName());
+ }
+ }
+ }
+ logger.info("After copying exit lists, there are still "
+ + fileNamesInRsync.size() + " files left in "
+ + rsyncDirectory.getAbsolutePath() + ".");
+
+ /* Copy Torperf files. */
+ if (torperfOutputDirectory != null) {
+ files.add(torperfOutputDirectory);
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else if (pop.getName().endsWith(".tpf") &&
+ pop.lastModified() >= cutOffMillis) {
+ String fileName = pop.getName();
+ this.copyFile(pop, new File(rsyncDirectory,
+ "torperf/" + fileName));
+ fileNamesInRsync.remove(pop.getName());
+ }
+ }
+ }
+ logger.info("After copying Torperf files, there are still "
+ + fileNamesInRsync.size() + " files left in "
+ + rsyncDirectory.getAbsolutePath() + ".");
+
+ /* Delete all files that we didn't (over-)write in this run. */
+ files.add(rsyncDirectory);
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else if (fileNamesInRsync.contains(pop.getName())) {
+ fileNamesInRsync.remove(pop.getName());
+ pop.delete();
+ }
+ }
+ logger.info("After deleting files that we didn't overwrite in this "
+ + "run, there are " + fileNamesInRsync.size() + " files left in "
+ + rsyncDirectory.getAbsolutePath() + ".");
+ }
+
+ private void copyFile(File from, File to) {
+ if (from.exists() && to.exists() &&
+ from.lastModified() == to.lastModified() &&
+ from.length() == to.length()) {
+ return;
+ }
+ try {
+ to.getParentFile().mkdirs();
+ FileInputStream fis = new FileInputStream(from);
+ BufferedInputStream bis = new BufferedInputStream(fis);
+ FileOutputStream fos = new FileOutputStream(to);
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ fos.write(data, 0, len);
+ }
+ bis.close();
+ fos.close();
+ to.setLastModified(from.lastModified());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveReader.java b/src/org/torproject/ernie/db/relaydescs/ArchiveReader.java
new file mode 100644
index 0000000..fba0a9f
--- /dev/null
+++ b/src/org/torproject/ernie/db/relaydescs/ArchiveReader.java
@@ -0,0 +1,146 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.relaydescs;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.Stack;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+
+/**
+ * Read in all files in a given directory and pass buffered readers of
+ * them to the relay descriptor parser.
+ */
+public class ArchiveReader {
+ public ArchiveReader(RelayDescriptorParser rdp, File archivesDirectory,
+ File statsDirectory, boolean keepImportHistory) {
+
+ if (rdp == null || archivesDirectory == null ||
+ statsDirectory == null) {
+ throw new IllegalArgumentException();
+ }
+
+ int parsedFiles = 0, ignoredFiles = 0;
+ Logger logger = Logger.getLogger(ArchiveReader.class.getName());
+ SortedSet<String> archivesImportHistory = new TreeSet<String>();
+ File archivesImportHistoryFile = new File(statsDirectory,
+ "archives-import-history");
+ if (keepImportHistory && archivesImportHistoryFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ archivesImportHistoryFile));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ archivesImportHistory.add(line);
+ }
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not read in archives import "
+ + "history file. Skipping.");
+ }
+ }
+ if (archivesDirectory.exists()) {
+ logger.fine("Importing files in directory " + archivesDirectory
+ + "/...");
+ Stack<File> filesInInputDir = new Stack<File>();
+ filesInInputDir.add(archivesDirectory);
+ List<File> problems = new ArrayList<File>();
+ while (!filesInInputDir.isEmpty()) {
+ File pop = filesInInputDir.pop();
+ if (pop.isDirectory()) {
+ for (File f : pop.listFiles()) {
+ filesInInputDir.add(f);
+ }
+ } else {
+ if (rdp != null) {
+ try {
+ BufferedInputStream bis = null;
+ if (keepImportHistory &&
+ archivesImportHistory.contains(pop.getName())) {
+ ignoredFiles++;
+ continue;
+ } else if (pop.getName().endsWith(".tar.bz2")) {
+ logger.warning("Cannot parse compressed tarball "
+ + pop.getAbsolutePath() + ". Skipping.");
+ continue;
+ } else if (pop.getName().endsWith(".bz2")) {
+ FileInputStream fis = new FileInputStream(pop);
+ BZip2CompressorInputStream bcis =
+ new BZip2CompressorInputStream(fis);
+ bis = new BufferedInputStream(bcis);
+ } else {
+ FileInputStream fis = new FileInputStream(pop);
+ bis = new BufferedInputStream(fis);
+ }
+ if (keepImportHistory) {
+ archivesImportHistory.add(pop.getName());
+ }
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ bis.close();
+ byte[] allData = baos.toByteArray();
+ rdp.parse(allData);
+ parsedFiles++;
+ } catch (IOException e) {
+ problems.add(pop);
+ if (problems.size() > 3) {
+ break;
+ }
+ }
+ }
+ }
+ }
+ if (problems.isEmpty()) {
+ logger.fine("Finished importing files in directory "
+ + archivesDirectory + "/.");
+ } else {
+ StringBuilder sb = new StringBuilder("Failed importing files in "
+ + "directory " + archivesDirectory + "/:");
+ int printed = 0;
+ for (File f : problems) {
+ sb.append("\n " + f.getAbsolutePath());
+ if (++printed >= 3) {
+ sb.append("\n ... more");
+ break;
+ }
+ }
+ }
+ }
+ if (keepImportHistory) {
+ try {
+ archivesImportHistoryFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ archivesImportHistoryFile));
+ for (String line : archivesImportHistory) {
+ bw.write(line + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not write archives import "
+ + "history file.");
+ }
+ }
+ logger.info("Finished importing relay descriptors from local "
+ + "directory:\nParsed " + parsedFiles + ", ignored "
+ + ignoredFiles + " files.");
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
new file mode 100644
index 0000000..c632656
--- /dev/null
+++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
@@ -0,0 +1,339 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.relaydescs;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.SortedSet;
+import java.util.Stack;
+import java.util.TimeZone;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.Hex;
+import org.torproject.descriptor.DescriptorParser;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.impl.DescriptorParseException;
+
+public class ArchiveWriter {
+ private Logger logger;
+ private File outputDirectory;
+ private DescriptorParser descriptorParser;
+ private int storedConsensuses = 0, storedVotes = 0, storedCerts = 0,
+ storedServerDescriptors = 0, storedExtraInfoDescriptors = 0;
+
+ public ArchiveWriter(File outputDirectory) {
+
+ if (outputDirectory == null) {
+ throw new IllegalArgumentException();
+ }
+
+ this.logger = Logger.getLogger(ArchiveWriter.class.getName());
+ this.outputDirectory = outputDirectory;
+ this.descriptorParser =
+ DescriptorSourceFactory.createDescriptorParser();
+ }
+
+ private boolean store(byte[] typeAnnotation, byte[] data,
+ String filename) {
+ try {
+ File file = new File(filename);
+ if (!file.exists()) {
+ this.logger.finer("Storing " + filename);
+ if (this.descriptorParser.parseDescriptors(data, filename).size()
+ != 1) {
+ this.logger.info("Relay descriptor file " + filename
+ + " doesn't contain exactly one descriptor. Not storing.");
+ return false;
+ }
+ file.getParentFile().mkdirs();
+ BufferedOutputStream bos = new BufferedOutputStream(
+ new FileOutputStream(file));
+ if (data.length > 0 && data[0] != '@') {
+ bos.write(typeAnnotation, 0, typeAnnotation.length);
+ }
+ bos.write(data, 0, data.length);
+ bos.close();
+ return true;
+ }
+ } catch (DescriptorParseException e) {
+ this.logger.log(Level.WARNING, "Could not parse relay descriptor "
+ + filename + " before storing it to disk. Skipping.", e);
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not store relay descriptor "
+ + filename, e);
+ }
+ return false;
+ }
+
+ private static final byte[] CONSENSUS_ANNOTATION =
+ "@type network-status-consensus-3 1.0\n".getBytes();
+ public void storeConsensus(byte[] data, long validAfter) {
+ SimpleDateFormat printFormat = new SimpleDateFormat(
+ "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String filename = outputDirectory + "/consensus/"
+ + printFormat.format(new Date(validAfter)) + "-consensus";
+ if (this.store(CONSENSUS_ANNOTATION, data, filename)) {
+ this.storedConsensuses++;
+ }
+ }
+
+ private static final byte[] VOTE_ANNOTATION =
+ "@type network-status-vote-3 1.0\n".getBytes();
+ public void storeVote(byte[] data, long validAfter,
+ String fingerprint, String digest) {
+ SimpleDateFormat printFormat = new SimpleDateFormat(
+ "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String filename = outputDirectory + "/vote/"
+ + printFormat.format(new Date(validAfter)) + "-vote-"
+ + fingerprint + "-" + digest;
+ if (this.store(VOTE_ANNOTATION, data, filename)) {
+ this.storedVotes++;
+ }
+ }
+
+ private static final byte[] CERTIFICATE_ANNOTATION =
+ "@type dir-key-certificate-3 1.0\n".getBytes();
+ public void storeCertificate(byte[] data, String fingerprint,
+ long published) {
+ SimpleDateFormat printFormat = new SimpleDateFormat(
+ "yyyy-MM-dd-HH-mm-ss");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String filename = outputDirectory + "/certs/"
+ + fingerprint + "-" + printFormat.format(new Date(published));
+ if (this.store(CERTIFICATE_ANNOTATION, data, filename)) {
+ this.storedCerts++;
+ }
+ }
+
+ private static final byte[] SERVER_DESCRIPTOR_ANNOTATION =
+ "@type server-descriptor 1.0\n".getBytes();
+ public void storeServerDescriptor(byte[] data, String digest,
+ long published) {
+ SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String filename = outputDirectory + "/server-descriptor/"
+ + printFormat.format(new Date(published))
+ + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
+ + digest;
+ if (this.store(SERVER_DESCRIPTOR_ANNOTATION, data, filename)) {
+ this.storedServerDescriptors++;
+ }
+ }
+
+ private static final byte[] EXTRA_INFO_ANNOTATION =
+ "@type extra-info 1.0\n".getBytes();
+ public void storeExtraInfoDescriptor(byte[] data,
+ String extraInfoDigest, long published) {
+ SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
+ descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String filename = outputDirectory + "/extra-info/"
+ + descriptorFormat.format(new Date(published))
+ + extraInfoDigest.substring(0, 1) + "/"
+ + extraInfoDigest.substring(1, 2) + "/"
+ + extraInfoDigest;
+ if (this.store(EXTRA_INFO_ANNOTATION, data, filename)) {
+ this.storedExtraInfoDescriptors++;
+ }
+ }
+
+ private StringBuilder intermediateStats = new StringBuilder();
+ public void intermediateStats(String event) {
+ intermediateStats.append("While " + event + ", we stored "
+ + this.storedConsensuses + " consensus(es), " + this.storedVotes
+ + " vote(s), " + this.storedCerts + " certificate(s), "
+ + this.storedServerDescriptors + " server descriptor(s), and "
+ + this.storedExtraInfoDescriptors
+ + " extra-info descriptor(s) to disk.\n");
+ this.storedConsensuses = 0;
+ this.storedVotes = 0;
+ this.storedCerts = 0;
+ this.storedServerDescriptors = 0;
+ this.storedExtraInfoDescriptors = 0;
+ }
+ /**
+ * Dump some statistics on the completeness of descriptors to the logs
+ * on level INFO.
+ */
+ public void dumpStats() {
+ StringBuilder sb = new StringBuilder("Finished writing relay "
+ + "descriptors to disk.\n");
+ sb.append(intermediateStats.toString());
+ sb.append("Statistics on the completeness of written relay "
+ + "descriptors of the last 3 consensuses (Consensus/Vote, "
+ + "valid-after, votes, server descriptors, extra-infos):");
+ try {
+ SimpleDateFormat validAfterFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ validAfterFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ SimpleDateFormat consensusVoteFormat =
+ new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+ consensusVoteFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ SimpleDateFormat descriptorFormat =
+ new SimpleDateFormat("yyyy/MM/");
+ descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+ SortedSet<File> consensuses = new TreeSet<File>();
+ Stack<File> leftToParse = new Stack<File>();
+ leftToParse.add(new File(outputDirectory + "/consensus"));
+ while (!leftToParse.isEmpty()) {
+ File pop = leftToParse.pop();
+ if (pop.isDirectory()) {
+ for (File f : pop.listFiles()) {
+ leftToParse.add(f);
+ }
+ } else if (pop.length() > 0) {
+ consensuses.add(pop);
+ }
+ while (consensuses.size() > 3) {
+ consensuses.remove(consensuses.first());
+ }
+ }
+ for (File f : consensuses) {
+ BufferedReader br = new BufferedReader(new FileReader(f));
+ String line = null, validAfterTime = null,
+ voteFilenamePrefix = null, dirSource = null;
+ int allVotes = 0, foundVotes = 0,
+ allServerDescs = 0, foundServerDescs = 0,
+ allExtraInfos = 0, foundExtraInfos = 0;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("valid-after ")) {
+ validAfterTime = line.substring("valid-after ".length());
+ long validAfter = validAfterFormat.parse(
+ validAfterTime).getTime();
+ voteFilenamePrefix = outputDirectory + "/vote/"
+ + consensusVoteFormat.format(new Date(validAfter))
+ + "-vote-";
+ } else if (line.startsWith("dir-source ")) {
+ dirSource = line.split(" ")[2];
+ } else if (line.startsWith("vote-digest ")) {
+ allVotes++;
+ File voteFile = new File(voteFilenamePrefix + dirSource + "-"
+ + line.split(" ")[1]);
+ if (voteFile.exists()) {
+ foundVotes++;
+ BufferedReader vbr = new BufferedReader(new FileReader(
+ voteFile));
+ String line3 = null;
+ int voteAllServerDescs = 0, voteFoundServerDescs = 0,
+ voteAllExtraInfos = 0, voteFoundExtraInfos = 0;
+ while ((line3 = vbr.readLine()) != null) {
+ if (line3.startsWith("r ")) {
+ voteAllServerDescs++;
+ String digest = Hex.encodeHexString(Base64.decodeBase64(
+ line3.split(" ")[3] + "=")).toLowerCase();
+ long published = validAfterFormat.parse(
+ line3.split(" ")[4] + " "
+ + line3.split(" ")[5]).getTime();
+ String filename = outputDirectory
+ + "/server-descriptor/"
+ + descriptorFormat.format(new Date(published))
+ + digest.substring(0, 1) + "/"
+ + digest.substring(1, 2) + "/" + digest;
+ if (new File(filename).exists()) {
+ BufferedReader sbr = new BufferedReader(new FileReader(
+ new File(filename)));
+ String line2 = null;
+ while ((line2 = sbr.readLine()) != null) {
+ if (line2.startsWith("opt extra-info-digest ") ||
+ line2.startsWith("extra-info-digest ")) {
+ voteAllExtraInfos++;
+ String extraInfoDigest = line2.startsWith("opt ") ?
+ line2.split(" ")[2].toLowerCase() :
+ line2.split(" ")[1].toLowerCase();
+ String filename2 =
+ outputDirectory.getAbsolutePath()
+ + "/extra-info/"
+ + descriptorFormat.format(new Date(published))
+ + extraInfoDigest.substring(0, 1) + "/"
+ + extraInfoDigest.substring(1, 2) + "/"
+ + extraInfoDigest;
+ if (new File(filename2).exists()) {
+ voteFoundExtraInfos++;
+ }
+ }
+ }
+ sbr.close();
+ voteFoundServerDescs++;
+ }
+ }
+ }
+ vbr.close();
+ sb.append(String.format("%nV, %s, NA, %d/%d (%.1f%%), "
+ + "%d/%d (%.1f%%)", validAfterTime,
+ voteFoundServerDescs, voteAllServerDescs,
+ 100.0D * (double) voteFoundServerDescs /
+ (double) voteAllServerDescs,
+ voteFoundExtraInfos, voteAllExtraInfos,
+ 100.0D * (double) voteFoundExtraInfos /
+ (double) voteAllExtraInfos));
+ }
+ } else if (line.startsWith("r ")) {
+ allServerDescs++;
+ String digest = Hex.encodeHexString(Base64.decodeBase64(
+ line.split(" ")[3] + "=")).toLowerCase();
+ long published = validAfterFormat.parse(
+ line.split(" ")[4] + " " + line.split(" ")[5]).getTime();
+ String filename = outputDirectory.getAbsolutePath()
+ + "/server-descriptor/"
+ + descriptorFormat.format(new Date(published))
+ + digest.substring(0, 1) + "/"
+ + digest.substring(1, 2) + "/" + digest;
+ if (new File (filename).exists()) {
+ BufferedReader sbr = new BufferedReader(new FileReader(
+ new File(filename)));
+ String line2 = null;
+ while ((line2 = sbr.readLine()) != null) {
+ if (line2.startsWith("opt extra-info-digest ") ||
+ line2.startsWith("extra-info-digest ")) {
+ allExtraInfos++;
+ String extraInfoDigest = line2.startsWith("opt ") ?
+ line2.split(" ")[2].toLowerCase() :
+ line2.split(" ")[1].toLowerCase();
+ String filename2 = outputDirectory.getAbsolutePath()
+ + "/extra-info/"
+ + descriptorFormat.format(new Date(published))
+ + extraInfoDigest.substring(0, 1) + "/"
+ + extraInfoDigest.substring(1, 2) + "/"
+ + extraInfoDigest;
+ if (new File (filename2).exists()) {
+ foundExtraInfos++;
+ }
+ }
+ }
+ sbr.close();
+ foundServerDescs++;
+ }
+ }
+ }
+ br.close();
+ sb.append(String.format("%nC, %s, %d/%d (%.1f%%), "
+ + "%d/%d (%.1f%%), %d/%d (%.1f%%)",
+ validAfterTime, foundVotes, allVotes,
+ 100.0D * (double) foundVotes / (double) allVotes,
+ foundServerDescs, allServerDescs,
+ 100.0D * (double) foundServerDescs / (double) allServerDescs,
+ foundExtraInfos, allExtraInfos,
+ 100.0D * (double) foundExtraInfos / (double) allExtraInfos));
+ }
+ this.logger.info(sb.toString());
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not dump statistics to disk.",
+ e);
+ } catch (ParseException e) {
+ this.logger.log(Level.WARNING, "Could not dump statistics to disk.",
+ e);
+ }
+ }
+}
diff --git a/src/org/torproject/ernie/db/relaydescs/CachedRelayDescriptorReader.java b/src/org/torproject/ernie/db/relaydescs/CachedRelayDescriptorReader.java
new file mode 100644
index 0000000..194e0a2
--- /dev/null
+++ b/src/org/torproject/ernie/db/relaydescs/CachedRelayDescriptorReader.java
@@ -0,0 +1,235 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.relaydescs;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.StringReader;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+
+/**
+ * Parses all descriptors in local directory cacheddesc/ and sorts them
+ * into directory structure in directory-archive/.
+ */
+public class CachedRelayDescriptorReader {
+ public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
+ List<String> inputDirectories, File statsDirectory) {
+
+ if (rdp == null || inputDirectories == null ||
+ inputDirectories.isEmpty() || statsDirectory == null) {
+ throw new IllegalArgumentException();
+ }
+
+ StringBuilder dumpStats = new StringBuilder("Finished importing "
+ + "relay descriptors from local Tor data directories:");
+ Logger logger = Logger.getLogger(
+ CachedRelayDescriptorReader.class.getName());
+
+ /* Read import history containing SHA-1 digests of previously parsed
+ * statuses and descriptors, so that we can skip them in this run. */
+ Set<String> lastImportHistory = new HashSet<String>(),
+ currentImportHistory = new HashSet<String>();
+ File importHistoryFile = new File(statsDirectory,
+ "cacheddesc-import-history");
+ if (importHistoryFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ importHistoryFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ lastImportHistory.add(line);
+ }
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not read import history from "
+ + importHistoryFile.getAbsolutePath() + ".", e);
+ }
+ }
+
+ /* Read cached descriptors directories. */
+ for (String inputDirectory : inputDirectories) {
+ File cachedDescDir = new File(inputDirectory);
+ if (!cachedDescDir.exists()) {
+ logger.warning("Directory " + cachedDescDir.getAbsolutePath()
+ + " does not exist. Skipping.");
+ continue;
+ }
+ logger.fine("Reading " + cachedDescDir.getAbsolutePath()
+ + " directory.");
+ for (File f : cachedDescDir.listFiles()) {
+ try {
+ // descriptors may contain non-ASCII chars; read as bytes to
+ // determine digests
+ BufferedInputStream bis =
+ new BufferedInputStream(new FileInputStream(f));
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ bis.close();
+ byte[] allData = baos.toByteArray();
+ if (f.getName().equals("cached-consensus")) {
+ /* Check if directory information is stale. */
+ BufferedReader br = new BufferedReader(new StringReader(
+ new String(allData, "US-ASCII")));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("valid-after ")) {
+ dumpStats.append("\n" + f.getName() + ": " + line.substring(
+ "valid-after ".length()));
+ SimpleDateFormat dateTimeFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ if (dateTimeFormat.parse(line.substring("valid-after ".
+ length())).getTime() < System.currentTimeMillis()
+ - 6L * 60L * 60L * 1000L) {
+ logger.warning("Cached descriptor files in "
+ + cachedDescDir.getAbsolutePath() + " are stale. "
+ + "The valid-after line in cached-consensus is '"
+ + line + "'.");
+ dumpStats.append(" (stale!)");
+ }
+ break;
+ }
+ }
+ br.close();
+
+ /* Parse the cached consensus if we haven't parsed it before
+ * (but regardless of whether it's stale or not). */
+ if (rdp != null) {
+ String digest = Hex.encodeHexString(DigestUtils.sha(
+ allData));
+ if (!lastImportHistory.contains(digest) &&
+ !currentImportHistory.contains(digest)) {
+ rdp.parse(allData);
+ } else {
+ dumpStats.append(" (skipped)");
+ }
+ currentImportHistory.add(digest);
+ }
+ } else if (f.getName().equals("v3-status-votes")) {
+ int parsedNum = 0, skippedNum = 0;
+ String ascii = new String(allData, "US-ASCII");
+ String startToken = "network-status-version ";
+ int end = ascii.length();
+ int start = ascii.indexOf(startToken);
+ while (start >= 0 && start < end) {
+ int next = ascii.indexOf(startToken, start + 1);
+ if (next < 0) {
+ next = end;
+ }
+ if (start < next) {
+ byte[] rawNetworkStatusBytes = new byte[next - start];
+ System.arraycopy(allData, start, rawNetworkStatusBytes, 0,
+ next - start);
+ if (rdp != null) {
+ String digest = Hex.encodeHexString(DigestUtils.sha(
+ rawNetworkStatusBytes));
+ if (!lastImportHistory.contains(digest) &&
+ !currentImportHistory.contains(digest)) {
+ rdp.parse(rawNetworkStatusBytes);
+ parsedNum++;
+ } else {
+ skippedNum++;
+ }
+ currentImportHistory.add(digest);
+ }
+ }
+ start = next;
+ }
+ dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum
+ + ", skipped " + skippedNum + " votes");
+ } else if (f.getName().startsWith("cached-descriptors") ||
+ f.getName().startsWith("cached-extrainfo")) {
+ String ascii = new String(allData, "US-ASCII");
+ int start = -1, sig = -1, end = -1;
+ String startToken =
+ f.getName().startsWith("cached-descriptors") ?
+ "router " : "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ String endToken = "\n-----END SIGNATURE-----\n";
+ int parsedNum = 0, skippedNum = 0;
+ while (end < ascii.length()) {
+ start = ascii.indexOf(startToken, end);
+ if (start < 0) {
+ break;
+ }
+ sig = ascii.indexOf(sigToken, start);
+ if (sig < 0) {
+ break;
+ }
+ sig += sigToken.length();
+ end = ascii.indexOf(endToken, sig);
+ if (end < 0) {
+ break;
+ }
+ end += endToken.length();
+ byte[] descBytes = new byte[end - start];
+ System.arraycopy(allData, start, descBytes, 0, end - start);
+ if (rdp != null) {
+ String digest = Hex.encodeHexString(DigestUtils.sha(
+ descBytes));
+ if (!lastImportHistory.contains(digest) &&
+ !currentImportHistory.contains(digest)) {
+ rdp.parse(descBytes);
+ parsedNum++;
+ } else {
+ skippedNum++;
+ }
+ currentImportHistory.add(digest);
+ }
+ }
+ dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum
+ + ", skipped " + skippedNum + " "
+ + (f.getName().startsWith("cached-descriptors") ?
+ "server" : "extra-info") + " descriptors");
+ }
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed reading "
+ + cachedDescDir.getAbsolutePath() + " directory.", e);
+ } catch (ParseException e) {
+ logger.log(Level.WARNING, "Failed reading "
+ + cachedDescDir.getAbsolutePath() + " directory.", e);
+ }
+ }
+ logger.fine("Finished reading "
+ + cachedDescDir.getAbsolutePath() + " directory.");
+ }
+
+ /* Write import history containing SHA-1 digests to disk. */
+ try {
+ importHistoryFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ importHistoryFile));
+ for (String digest : currentImportHistory) {
+ bw.write(digest + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not write import history to "
+ + importHistoryFile.getAbsolutePath() + ".", e);
+ }
+
+ logger.info(dumpStats.toString());
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/relaydescs/RelayDescriptorDownloader.java b/src/org/torproject/ernie/db/relaydescs/RelayDescriptorDownloader.java
new file mode 100644
index 0000000..0bea50a
--- /dev/null
+++ b/src/org/torproject/ernie/db/relaydescs/RelayDescriptorDownloader.java
@@ -0,0 +1,821 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.relaydescs;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.zip.InflaterInputStream;
+
+/**
+ * Downloads relay descriptors from the directory authorities via HTTP.
+ * Keeps a list of missing descriptors that gets updated by parse results
+ * from <code>RelayDescriptorParser</code> and downloads all missing
+ * descriptors that have been published in the last 24 hours. Also
+ * downloads all server and extra-info descriptors known to a directory
+ * authority at most once a day.
+ */
+public class RelayDescriptorDownloader {
+
+ /**
+ * Text file containing the descriptors that we are missing and that we
+ * want to download. Lines are formatted as:
+ *
+ * - "consensus,<validafter>,<parsed>",
+ * - "vote,<validafter>,<fingerprint>,<parsed>",
+ * - "server,<published>,<relayid>,<descid>,<parsed>", or
+ * - "extra,<published>,<relayid>,<descid>,<parsed>".
+ */
+ private File missingDescriptorsFile;
+
+ /**
+ * Relay descriptors that we are missing and that we want to download
+ * either in this execution or write to disk and try next time. Map keys
+ * contain comma-separated values as in the missing descriptors files
+ * without the "parsed" column. Map values contain the "parsed" column.
+ */
+ private SortedMap<String, String> missingDescriptors;
+
+ /**
+ * Text file containing the IP addresses (and Dir ports if not 80) of
+ * directory authorities and when we last downloaded all server and
+ * extra-info descriptors from them, so that we can avoid downloading
+ * them too often.
+ */
+ private File lastDownloadedAllDescriptorsFile;
+
+ /**
+ * Map of directory authorities and when we last downloaded all server
+ * and extra-info descriptors from them. Map keys are IP addresses (and
+ * Dir ports if not 80), map values are timestamps.
+ */
+ private Map<String, String> lastDownloadedAllDescriptors;
+
+ /**
+ * <code>RelayDescriptorParser</code> that we will hand over the
+ * downloaded descriptors for parsing.
+ */
+ private RelayDescriptorParser rdp;
+
+ /**
+ * Directory authorities that we will try to download missing
+ * descriptors from.
+ */
+ private List<String> authorities;
+
+ /**
+ * Should we try to download the current consensus if we don't have it?
+ */
+ private boolean downloadCurrentConsensus;
+
+ /**
+ * Should we try to download current votes if we don't have them?
+ */
+ private boolean downloadCurrentVotes;
+
+ /**
+ * Should we try to download missing server descriptors that have been
+ * published within the past 24 hours?
+ */
+ private boolean downloadMissingServerDescriptors;
+
+ /**
+ * Should we try to download missing extra-info descriptors that have
+ * been published within the past 24 hours?
+ */
+ private boolean downloadMissingExtraInfos;
+
+ /**
+ * Should we try to download all server descriptors from the authorities
+ * once every 24 hours?
+ */
+ private boolean downloadAllServerDescriptors;
+
+ /**
+ * Should we try to download all extra-info descriptors from the
+ * authorities once every 24 hours?
+ */
+ private boolean downloadAllExtraInfos;
+
+ /**
+ * Should we download zlib-compressed versions of descriptors by adding
+ * ".z" to URLs?
+ */
+ private boolean downloadCompressed;
+
+ /**
+ * valid-after time that we expect the current consensus and votes to
+ * have, formatted "yyyy-MM-dd HH:mm:ss". We only expect to find
+ * consensuses and votes with this valid-after time on the directory
+ * authorities. This time is initialized as the beginning of the current
+ * hour.
+ */
+ private String currentValidAfter;
+
+ /**
+ * Cut-off time for missing server and extra-info descriptors, formatted
+ * "yyyy-MM-dd HH:mm:ss". This time is initialized as the current system
+ * time minus 24 hours.
+ */
+ private String descriptorCutOff;
+
+ /**
+ * Cut-off time for downloading all server and extra-info descriptors
+ * from the directory authorities, formatted "yyyy-MM-dd HH:mm:ss". This
+ * time is initialized as the current system time minus 23:30 hours.
+ */
+ private String downloadAllDescriptorsCutOff;
+
+ /**
+ * Directory authorities that we plan to download all server and
+ * extra-info descriptors from in this execution.
+ */
+ private Set<String> downloadAllDescriptorsFromAuthorities;
+
+ /**
+ * Current timestamp that is written to the missing list for descriptors
+ * that we parsed in this execution and for authorities that we
+ * downloaded all server and extra-info descriptors from.
+ */
+ private String currentTimestamp;
+
+ /**
+ * Logger for this class.
+ */
+ private Logger logger;
+
+ /**
+ * Number of descriptors requested by directory authority to be included
+ * in logs.
+ */
+ private Map<String, Integer> requestsByAuthority;
+
+ /**
+ * Counters for descriptors that we had on the missing list at the
+ * beginning of the execution, that we added to the missing list,
+ * that we requested, and that we successfully downloaded in this
+ * execution.
+ */
+ private int oldMissingConsensuses = 0, oldMissingVotes = 0,
+ oldMissingServerDescriptors = 0, oldMissingExtraInfoDescriptors = 0,
+ newMissingConsensuses = 0, newMissingVotes = 0,
+ newMissingServerDescriptors = 0, newMissingExtraInfoDescriptors = 0,
+ requestedConsensuses = 0, requestedVotes = 0,
+ requestedMissingServerDescriptors = 0,
+ requestedAllServerDescriptors = 0,
+ requestedMissingExtraInfoDescriptors = 0,
+ requestedAllExtraInfoDescriptors = 0, downloadedConsensuses = 0,
+ downloadedVotes = 0, downloadedMissingServerDescriptors = 0,
+ downloadedAllServerDescriptors = 0,
+ downloadedMissingExtraInfoDescriptors = 0,
+ downloadedAllExtraInfoDescriptors = 0;
+
+ /**
+ * Initializes this class, including reading in missing descriptors from
+ * <code>stats/missing-relay-descriptors</code> and the times when we
+ * last downloaded all server and extra-info descriptors from
+ * <code>stats/last-downloaded-all-descriptors</code>.
+ */
+ public RelayDescriptorDownloader(RelayDescriptorParser rdp,
+ List<String> authorities, boolean downloadCurrentConsensus,
+ boolean downloadCurrentVotes,
+ boolean downloadMissingServerDescriptors,
+ boolean downloadMissingExtraInfos,
+ boolean downloadAllServerDescriptors, boolean downloadAllExtraInfos,
+ boolean downloadCompressed) {
+
+ /* Memorize argument values. */
+ this.rdp = rdp;
+ this.authorities = new ArrayList<String>(authorities);
+ this.downloadCurrentConsensus = downloadCurrentConsensus;
+ this.downloadCurrentVotes = downloadCurrentVotes;
+ this.downloadMissingServerDescriptors =
+ downloadMissingServerDescriptors;
+ this.downloadMissingExtraInfos = downloadMissingExtraInfos;
+ this.downloadAllServerDescriptors = downloadAllServerDescriptors;
+ this.downloadAllExtraInfos = downloadAllExtraInfos;
+ this.downloadCompressed = downloadCompressed;
+
+ /* Shuffle list of authorities for better load balancing over time. */
+ Collections.shuffle(this.authorities);
+
+ /* Initialize logger. */
+ this.logger = Logger.getLogger(
+ RelayDescriptorDownloader.class.getName());
+
+ /* Prepare cut-off times and timestamp for the missing descriptors
+ * list and the list of authorities to download all server and
+ * extra-info descriptors from. */
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long now = System.currentTimeMillis();
+ this.currentValidAfter = format.format((now / (60L * 60L * 1000L)) *
+ (60L * 60L * 1000L));
+ this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L);
+ this.currentTimestamp = format.format(now);
+ this.downloadAllDescriptorsCutOff = format.format(now
+ - 23L * 60L * 60L * 1000L - 30L * 60L * 1000L);
+
+ /* Read list of missing descriptors from disk and memorize those that
+ * we are interested in and that are likely to be found on the
+ * directory authorities. */
+ this.missingDescriptors = new TreeMap<String, String>();
+ this.missingDescriptorsFile = new File(
+ "stats/missing-relay-descriptors");
+ if (this.missingDescriptorsFile.exists()) {
+ try {
+ this.logger.fine("Reading file "
+ + this.missingDescriptorsFile.getAbsolutePath() + "...");
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.missingDescriptorsFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.split(",").length > 2) {
+ String published = line.split(",")[1];
+ if (((line.startsWith("consensus,") ||
+ line.startsWith("vote,")) &&
+ this.currentValidAfter.equals(published)) ||
+ ((line.startsWith("server,") ||
+ line.startsWith("extra,")) &&
+ this.descriptorCutOff.compareTo(published) < 0)) {
+ if (!line.endsWith("NA")) {
+ /* Not missing. */
+ } else if (line.startsWith("consensus,")) {
+ oldMissingConsensuses++;
+ } else if (line.startsWith("vote,")) {
+ oldMissingVotes++;
+ } else if (line.startsWith("server,")) {
+ oldMissingServerDescriptors++;
+ } else if (line.startsWith("extra,")) {
+ oldMissingExtraInfoDescriptors++;
+ }
+ int separateAt = line.lastIndexOf(",");
+ this.missingDescriptors.put(line.substring(0,
+ separateAt), line.substring(separateAt + 1));
+ }
+ } else {
+ this.logger.fine("Invalid line '" + line + "' in "
+ + this.missingDescriptorsFile.getAbsolutePath()
+ + ". Ignoring.");
+ }
+ }
+ br.close();
+ this.logger.fine("Finished reading file "
+ + this.missingDescriptorsFile.getAbsolutePath() + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed to read file "
+ + this.missingDescriptorsFile.getAbsolutePath()
+ + "! This means that we might forget to dowload relay "
+ + "descriptors we are missing.", e);
+ }
+ }
+
+ /* Read list of directory authorities and when we last downloaded all
+ * server and extra-info descriptors from them. */
+ this.lastDownloadedAllDescriptors = new HashMap<String, String>();
+ this.lastDownloadedAllDescriptorsFile = new File(
+ "stats/last-downloaded-all-descriptors");
+ if (this.lastDownloadedAllDescriptorsFile.exists()) {
+ try {
+ this.logger.fine("Reading file "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+ + "...");
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.lastDownloadedAllDescriptorsFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.split(",").length != 2) {
+ this.logger.fine("Invalid line '" + line + "' in "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+ + ". Ignoring.");
+ } else {
+ String[] parts = line.split(",");
+ String authority = parts[0];
+ String lastDownloaded = parts[1];
+ this.lastDownloadedAllDescriptors.put(authority,
+ lastDownloaded);
+ }
+ }
+ br.close();
+ this.logger.fine("Finished reading file "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+ + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed to read file "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+ + "! This means that we might download all server and "
+ + "extra-info descriptors more often than we should.", e);
+ }
+ }
+
+ /* Make a list of at most two directory authorities that we want to
+ * download all server and extra-info descriptors from. */
+ this.downloadAllDescriptorsFromAuthorities = new HashSet<String>();
+ for (String authority : this.authorities) {
+ if (!this.lastDownloadedAllDescriptors.containsKey(authority) ||
+ this.lastDownloadedAllDescriptors.get(authority).compareTo(
+ this.downloadAllDescriptorsCutOff) < 0) {
+ this.downloadAllDescriptorsFromAuthorities.add(authority);
+ }
+ if (this.downloadAllDescriptorsFromAuthorities.size() >= 2) {
+ break;
+ }
+ }
+
+ /* Prepare statistics on this execution. */
+ this.requestsByAuthority = new HashMap<String, Integer>();
+ for (String authority : this.authorities) {
+ this.requestsByAuthority.put(authority, 0);
+ }
+ }
+
+ /**
+ * We have parsed a consensus. Take this consensus off the missing list
+ * and add the votes created by the given <code>authorities</code> and
+ * the <code>serverDescriptors</code> which are in the format
+ * "<published>,<relayid>,<descid>" to that list.
+ */
+ public void haveParsedConsensus(String validAfter,
+ Set<String> authorities, Set<String> serverDescriptors) {
+
+ /* Mark consensus as parsed. */
+ if (this.currentValidAfter.equals(validAfter)) {
+ String consensusKey = "consensus," + validAfter;
+ this.missingDescriptors.put(consensusKey, this.currentTimestamp);
+
+ /* Add votes to missing list. */
+ for (String authority : authorities) {
+ String voteKey = "vote," + validAfter + "," + authority;
+ if (!this.missingDescriptors.containsKey(voteKey)) {
+ this.missingDescriptors.put(voteKey, "NA");
+ this.newMissingVotes++;
+ }
+ }
+ }
+
+ /* Add server descriptors to missing list. */
+ for (String serverDescriptor : serverDescriptors) {
+ String published = serverDescriptor.split(",")[0];
+ if (this.descriptorCutOff.compareTo(published) < 0) {
+ String serverDescriptorKey = "server," + serverDescriptor;
+ if (!this.missingDescriptors.containsKey(
+ serverDescriptorKey)) {
+ this.missingDescriptors.put(serverDescriptorKey, "NA");
+ this.newMissingServerDescriptors++;
+ }
+ }
+ }
+ }
+
+ /**
+ * We have parsed a vote. Take this vote off the missing list and add
+ * the <code>serverDescriptors</code> which are in the format
+ * "<published>,<relayid>,<descid>" to that list.
+ */
+ public void haveParsedVote(String validAfter, String fingerprint,
+ Set<String> serverDescriptors) {
+
+ /* Mark vote as parsed. */
+ if (this.currentValidAfter.equals(validAfter)) {
+ String voteKey = "vote," + validAfter + "," + fingerprint;
+ this.missingDescriptors.put(voteKey, this.currentTimestamp);
+ }
+
+ /* Add server descriptors to missing list. */
+ for (String serverDescriptor : serverDescriptors) {
+ String published = serverDescriptor.split(",")[0];
+ if (this.descriptorCutOff.compareTo(published) < 0) {
+ String serverDescriptorKey = "server," + serverDescriptor;
+ if (!this.missingDescriptors.containsKey(
+ serverDescriptorKey)) {
+ this.missingDescriptors.put(serverDescriptorKey, "NA");
+ this.newMissingServerDescriptors++;
+ }
+ }
+ }
+ }
+
+ /**
+ * We have parsed a server descriptor. Take this server descriptor off
+ * the missing list and put the extra-info descriptor digest on that
+ * list.
+ */
+ public void haveParsedServerDescriptor(String published,
+ String relayIdentity, String serverDescriptorDigest,
+ String extraInfoDigest) {
+
+ /* Mark server descriptor as parsed. */
+ if (this.descriptorCutOff.compareTo(published) < 0) {
+ String serverDescriptorKey = "server," + published + ","
+ + relayIdentity + "," + serverDescriptorDigest;
+ this.missingDescriptors.put(serverDescriptorKey,
+ this.currentTimestamp);
+
+ /* Add extra-info descriptor to missing list. */
+ if (extraInfoDigest != null) {
+ String extraInfoKey = "extra," + published + ","
+ + relayIdentity + "," + extraInfoDigest;
+ if (!this.missingDescriptors.containsKey(extraInfoKey)) {
+ this.missingDescriptors.put(extraInfoKey, "NA");
+ this.newMissingExtraInfoDescriptors++;
+ }
+ }
+ }
+ }
+
+ /**
+ * We have parsed an extra-info descriptor. Take it off the missing
+ * list.
+ */
+ public void haveParsedExtraInfoDescriptor(String published,
+ String relayIdentity, String extraInfoDigest) {
+ if (this.descriptorCutOff.compareTo(published) < 0) {
+ String extraInfoKey = "extra," + published + ","
+ + relayIdentity + "," + extraInfoDigest;
+ this.missingDescriptors.put(extraInfoKey, this.currentTimestamp);
+ }
+ }
+
+ /**
+ * Downloads missing descriptors that we think might still be available
+ * on the directory authorities as well as all server and extra-info
+ * descriptors once per day.
+ */
+ public void downloadDescriptors() {
+
+ /* Put the current consensus on the missing list, unless we already
+ * have it. */
+ String consensusKey = "consensus," + this.currentValidAfter;
+ if (!this.missingDescriptors.containsKey(consensusKey)) {
+ this.missingDescriptors.put(consensusKey, "NA");
+ this.newMissingConsensuses++;
+ }
+
+ /* Download descriptors from authorities which are in random order, so
+ * that we distribute the load somewhat fairly over time. */
+ for (String authority : authorities) {
+
+ /* Make all requests to an authority in a single try block. If
+ * something goes wrong with this authority, we give up on all
+ * downloads and continue with the next authority. */
+ /* TODO Some authorities provide very little bandwidth and could
+ * slow down the entire download process. Ponder adding a timeout of
+ * 3 or 5 minutes per authority to avoid getting in the way of the
+ * next execution. */
+ try {
+
+ /* Start with downloading the current consensus, unless we already
+ * have it. */
+ if (downloadCurrentConsensus) {
+ if (this.missingDescriptors.containsKey(consensusKey) &&
+ this.missingDescriptors.get(consensusKey).equals("NA")) {
+ this.requestedConsensuses++;
+ this.downloadedConsensuses +=
+ this.downloadResourceFromAuthority(authority,
+ "/tor/status-vote/current/consensus");
+ }
+ }
+
+ /* Next, try to download current votes that we're missing. */
+ if (downloadCurrentVotes) {
+ String voteKeyPrefix = "vote," + this.currentValidAfter;
+ SortedSet<String> fingerprints = new TreeSet<String>();
+ for (Map.Entry<String, String> e :
+ this.missingDescriptors.entrySet()) {
+ if (e.getValue().equals("NA") &&
+ e.getKey().startsWith(voteKeyPrefix)) {
+ String fingerprint = e.getKey().split(",")[2];
+ fingerprints.add(fingerprint);
+ }
+ }
+ for (String fingerprint : fingerprints) {
+ this.requestedVotes++;
+ this.downloadedVotes +=
+ this.downloadResourceFromAuthority(authority,
+ "/tor/status-vote/current/" + fingerprint);
+ }
+ }
+
+ /* Download either all server and extra-info descriptors or only
+ * those that we're missing. Start with server descriptors, then
+ * request extra-info descriptors. */
+ List<String> types = new ArrayList<String>(Arrays.asList(
+ "server,extra".split(",")));
+ for (String type : types) {
+
+ /* Download all server or extra-info descriptors from this
+ * authority if we haven't done so for 24 hours and if we're
+ * configured to do so. */
+ if (this.downloadAllDescriptorsFromAuthorities.contains(
+ authority) && ((type.equals("server") &&
+ this.downloadAllServerDescriptors) ||
+ (type.equals("extra") && this.downloadAllExtraInfos))) {
+ int downloadedAllDescriptors =
+ this.downloadResourceFromAuthority(authority, "/tor/"
+ + type + "/all");
+ if (type.equals("server")) {
+ this.requestedAllServerDescriptors++;
+ this.downloadedAllServerDescriptors +=
+ downloadedAllDescriptors;
+ } else {
+ this.requestedAllExtraInfoDescriptors++;
+ this.downloadedAllExtraInfoDescriptors +=
+ downloadedAllDescriptors;
+ }
+
+ /* Download missing server or extra-info descriptors if we're
+ * configured to do so. */
+ } else if ((type.equals("server") &&
+ this.downloadMissingServerDescriptors) ||
+ (type.equals("extra") && this.downloadMissingExtraInfos)) {
+
+ /* Go through the list of missing descriptors of this type
+ * and combine the descriptor identifiers to a URL of up to
+ * 96 descriptors that we can download at once. */
+ SortedSet<String> descriptorIdentifiers =
+ new TreeSet<String>();
+ for (Map.Entry<String, String> e :
+ this.missingDescriptors.entrySet()) {
+ if (e.getValue().equals("NA") &&
+ e.getKey().startsWith(type + ",") &&
+ this.descriptorCutOff.compareTo(
+ e.getKey().split(",")[1]) < 0) {
+ String descriptorIdentifier = e.getKey().split(",")[3];
+ descriptorIdentifiers.add(descriptorIdentifier);
+ }
+ }
+ StringBuilder combinedResource = null;
+ int descriptorsInCombinedResource = 0,
+ requestedDescriptors = 0, downloadedDescriptors = 0;
+ for (String descriptorIdentifier : descriptorIdentifiers) {
+ if (descriptorsInCombinedResource >= 96) {
+ requestedDescriptors += descriptorsInCombinedResource;
+ downloadedDescriptors +=
+ this.downloadResourceFromAuthority(authority,
+ combinedResource.toString());
+ combinedResource = null;
+ descriptorsInCombinedResource = 0;
+ }
+ if (descriptorsInCombinedResource == 0) {
+ combinedResource = new StringBuilder("/tor/" + type
+ + "/d/" + descriptorIdentifier);
+ } else {
+ combinedResource.append("+" + descriptorIdentifier);
+ }
+ descriptorsInCombinedResource++;
+ }
+ if (descriptorsInCombinedResource > 0) {
+ requestedDescriptors += descriptorsInCombinedResource;
+ downloadedDescriptors +=
+ this.downloadResourceFromAuthority(authority,
+ combinedResource.toString());
+ }
+ if (type.equals("server")) {
+ this.requestedMissingServerDescriptors +=
+ requestedDescriptors;
+ this.downloadedMissingServerDescriptors +=
+ downloadedDescriptors;
+ } else {
+ this.requestedMissingExtraInfoDescriptors +=
+ requestedDescriptors;
+ this.downloadedMissingExtraInfoDescriptors +=
+ downloadedDescriptors;
+ }
+ }
+ }
+
+ /* If a download failed, stop requesting descriptors from this
+ * authority and move on to the next. */
+ } catch (IOException e) {
+ logger.log(Level.FINE, "Failed downloading from " + authority
+ + "!", e);
+ }
+ }
+ }
+
+ /**
+ * Attempts to download one or more descriptors identified by a resource
+ * string from a directory authority and passes the returned
+ * descriptor(s) to the <code>RelayDescriptorParser</code> upon success.
+ * Returns the number of descriptors contained in the reply. Throws an
+ * <code>IOException</code> if something goes wrong while downloading.
+ */
+ private int downloadResourceFromAuthority(String authority,
+ String resource) throws IOException {
+ byte[] allData = null;
+ this.requestsByAuthority.put(authority,
+ this.requestsByAuthority.get(authority) + 1);
+ /* TODO Disable compressed downloads for extra-info descriptors,
+ * because zlib decompression doesn't work correctly. Figure out why
+ * this is and fix it. */
+ String fullUrl = "http://"; + authority + resource
+ + (this.downloadCompressed && !resource.startsWith("/tor/extra/")
+ ? ".z" : "");
+ URL u = new URL(fullUrl);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ int response = huc.getResponseCode();
+ if (response == 200) {
+ BufferedInputStream in = this.downloadCompressed &&
+ !resource.startsWith("/tor/extra/")
+ ? new BufferedInputStream(new InflaterInputStream(
+ huc.getInputStream()))
+ : new BufferedInputStream(huc.getInputStream());
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = in.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ in.close();
+ allData = baos.toByteArray();
+ }
+ logger.fine("Downloaded " + fullUrl + " -> " + response + " ("
+ + (allData == null ? 0 : allData.length) + " bytes)");
+ int receivedDescriptors = 0;
+ if (allData != null) {
+ if (resource.startsWith("/tor/status-vote/current/")) {
+ this.rdp.parse(allData);
+ receivedDescriptors = 1;
+ } else if (resource.startsWith("/tor/server/") ||
+ resource.startsWith("/tor/extra/")) {
+ if (resource.equals("/tor/server/all")) {
+ this.lastDownloadedAllDescriptors.put(authority,
+ this.currentTimestamp);
+ }
+ String ascii = null;
+ try {
+ ascii = new String(allData, "US-ASCII");
+ } catch (UnsupportedEncodingException e) {
+ /* No way that US-ASCII is not supported. */
+ }
+ int start = -1, sig = -1, end = -1;
+ String startToken = resource.startsWith("/tor/server/") ?
+ "router " : "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ String endToken = "\n-----END SIGNATURE-----\n";
+ while (end < ascii.length()) {
+ start = ascii.indexOf(startToken, end);
+ if (start < 0) {
+ break;
+ }
+ sig = ascii.indexOf(sigToken, start);
+ if (sig < 0) {
+ break;
+ }
+ sig += sigToken.length();
+ end = ascii.indexOf(endToken, sig);
+ if (end < 0) {
+ break;
+ }
+ end += endToken.length();
+ byte[] descBytes = new byte[end - start];
+ System.arraycopy(allData, start, descBytes, 0, end - start);
+ this.rdp.parse(descBytes);
+ receivedDescriptors++;
+ }
+ }
+ }
+ return receivedDescriptors;
+ }
+
+ /**
+ * Writes status files to disk and logs statistics about downloading
+ * relay descriptors in this execution.
+ */
+ public void writeFile() {
+
+ /* Write missing descriptors file to disk. */
+ int missingConsensuses = 0, missingVotes = 0,
+ missingServerDescriptors = 0, missingExtraInfoDescriptors = 0;
+ try {
+ this.logger.fine("Writing file "
+ + this.missingDescriptorsFile.getAbsolutePath() + "...");
+ this.missingDescriptorsFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.missingDescriptorsFile));
+ for (Map.Entry<String, String> e :
+ this.missingDescriptors.entrySet()) {
+ String key = e.getKey(), value = e.getValue();
+ if (!value.equals("NA")) {
+ /* Not missing. */
+ } else if (key.startsWith("consensus,")) {
+ missingConsensuses++;
+ } else if (key.startsWith("vote,")) {
+ missingVotes++;
+ } else if (key.startsWith("server,")) {
+ missingServerDescriptors++;
+ } else if (key.startsWith("extra,")) {
+ missingExtraInfoDescriptors++;
+ }
+ bw.write(key + "," + value + "\n");
+ }
+ bw.close();
+ this.logger.fine("Finished writing file "
+ + this.missingDescriptorsFile.getAbsolutePath() + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed writing "
+ + this.missingDescriptorsFile.getAbsolutePath() + "!", e);
+ }
+
+ /* Write text file containing the directory authorities and when we
+ * last downloaded all server and extra-info descriptors from them to
+ * disk. */
+ try {
+ this.logger.fine("Writing file "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+ + "...");
+ this.lastDownloadedAllDescriptorsFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.lastDownloadedAllDescriptorsFile));
+ for (Map.Entry<String, String> e :
+ this.lastDownloadedAllDescriptors.entrySet()) {
+ String authority = e.getKey();
+ String lastDownloaded = e.getValue();
+ bw.write(authority + "," + lastDownloaded + "\n");
+ }
+ bw.close();
+ this.logger.fine("Finished writing file "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+ + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed writing "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + "!",
+ e);
+ }
+
+ /* Log statistics about this execution. */
+ this.logger.info("Finished downloading relay descriptors from the "
+ + "directory authorities.");
+ this.logger.info("At the beginning of this execution, we were "
+ + "missing " + oldMissingConsensuses + " consensus(es), "
+ + oldMissingVotes + " vote(s), " + oldMissingServerDescriptors
+ + " server descriptor(s), and " + oldMissingExtraInfoDescriptors
+ + " extra-info descriptor(s).");
+ this.logger.info("During this execution, we added "
+ + this.newMissingConsensuses + " consensus(es), "
+ + this.newMissingVotes + " vote(s), "
+ + this.newMissingServerDescriptors + " server descriptor(s), and "
+ + this.newMissingExtraInfoDescriptors + " extra-info "
+ + "descriptor(s) to the missing list, some of which we also "
+ + "requested and removed from the list again.");
+ this.logger.info("We requested " + this.requestedConsensuses
+ + " consensus(es), " + this.requestedVotes + " vote(s), "
+ + this.requestedMissingServerDescriptors + " missing server "
+ + "descriptor(s), " + this.requestedAllServerDescriptors
+ + " times all server descriptors, "
+ + this.requestedMissingExtraInfoDescriptors + " missing "
+ + "extra-info descriptor(s), and "
+ + this.requestedAllExtraInfoDescriptors + " times all extra-info "
+ + "descriptors from the directory authorities.");
+ StringBuilder sb = new StringBuilder();
+ for (String authority : this.authorities) {
+ sb.append(" " + authority + "="
+ + this.requestsByAuthority.get(authority));
+ }
+ this.logger.info("We sent these numbers of requests to the directory "
+ + "authorities:" + sb.toString());
+ this.logger.info("We successfully downloaded "
+ + this.downloadedConsensuses + " consensus(es), "
+ + this.downloadedVotes + " vote(s), "
+ + this.downloadedMissingServerDescriptors + " missing server "
+ + "descriptor(s), " + this.downloadedAllServerDescriptors
+ + " server descriptor(s) when downloading all descriptors, "
+ + this.downloadedMissingExtraInfoDescriptors + " missing "
+ + "extra-info descriptor(s) and "
+ + this.downloadedAllExtraInfoDescriptors + " extra-info "
+ + "descriptor(s) when downloading all descriptors.");
+ this.logger.info("At the end of this execution, we are missing "
+ + missingConsensuses + " consensus(es), " + missingVotes
+ + " vote(s), " + missingServerDescriptors + " server "
+ + "descriptor(s), and " + missingExtraInfoDescriptors
+ + " extra-info descriptor(s), some of which we may try in the next "
+ + "execution.");
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/relaydescs/RelayDescriptorParser.java b/src/org/torproject/ernie/db/relaydescs/RelayDescriptorParser.java
new file mode 100644
index 0000000..6f04c20
--- /dev/null
+++ b/src/org/torproject/ernie/db/relaydescs/RelayDescriptorParser.java
@@ -0,0 +1,265 @@
+/* Copyright 2010--2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.relaydescs;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+
+/**
+ * Parses relay descriptors including network status consensuses and
+ * votes, server and extra-info descriptors, and passes the results to the
+ * stats handlers, to the archive writer, or to the relay descriptor
+ * downloader.
+ */
+public class RelayDescriptorParser {
+
+ /**
+ * File writer that writes descriptor contents to files in a
+ * directory-archive directory structure.
+ */
+ private ArchiveWriter aw;
+
+ /**
+ * Missing descriptor downloader that uses the parse results to learn
+ * which descriptors we are missing and want to download.
+ */
+ private RelayDescriptorDownloader rdd;
+
+ /**
+ * Logger for this class.
+ */
+ private Logger logger;
+
+ private SimpleDateFormat dateTimeFormat;
+
+ /**
+ * Initializes this class.
+ */
+ public RelayDescriptorParser(ArchiveWriter aw) {
+ this.aw = aw;
+
+ /* Initialize logger. */
+ this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
+
+ this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ }
+
+ public void setRelayDescriptorDownloader(
+ RelayDescriptorDownloader rdd) {
+ this.rdd = rdd;
+ }
+
+ public void parse(byte[] data) {
+ try {
+ /* Convert descriptor to ASCII for parsing. This means we'll lose
+ * the non-ASCII chars, but we don't care about them for parsing
+ * anyway. */
+ BufferedReader br = new BufferedReader(new StringReader(new String(
+ data, "US-ASCII")));
+ String line;
+ do {
+ line = br.readLine();
+ } while (line != null && line.startsWith("@"));
+ if (line == null) {
+ this.logger.fine("We were given an empty descriptor for "
+ + "parsing. Ignoring.");
+ return;
+ }
+ SimpleDateFormat parseFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ if (line.equals("network-status-version 3")) {
+ // TODO when parsing the current consensus, check the fresh-until
+ // time to see when we switch from hourly to half-hourly
+ // consensuses
+ boolean isConsensus = true;
+ String validAfterTime = null, fingerprint = null,
+ dirSource = null;
+ long validAfter = -1L, dirKeyPublished = -1L;
+ SortedSet<String> dirSources = new TreeSet<String>();
+ SortedSet<String> serverDescriptors = new TreeSet<String>();
+ SortedSet<String> hashedRelayIdentities = new TreeSet<String>();
+ StringBuilder certificateStringBuilder = null;
+ String certificateString = null;
+ while ((line = br.readLine()) != null) {
+ if (certificateStringBuilder != null) {
+ if (line.startsWith("r ")) {
+ certificateString = certificateStringBuilder.toString();
+ certificateStringBuilder = null;
+ } else {
+ certificateStringBuilder.append(line + "\n");
+ }
+ }
+ if (line.equals("vote-status vote")) {
+ isConsensus = false;
+ } else if (line.startsWith("valid-after ")) {
+ validAfterTime = line.substring("valid-after ".length());
+ validAfter = parseFormat.parse(validAfterTime).getTime();
+ } else if (line.startsWith("dir-source ")) {
+ dirSource = line.split(" ")[2];
+ } else if (line.startsWith("vote-digest ")) {
+ dirSources.add(dirSource);
+ } else if (line.startsWith("dir-key-certificate-version ")) {
+ certificateStringBuilder = new StringBuilder();
+ certificateStringBuilder.append(line + "\n");
+ } else if (line.startsWith("fingerprint ")) {
+ fingerprint = line.split(" ")[1];
+ } else if (line.startsWith("dir-key-published ")) {
+ String dirKeyPublishedTime = line.substring(
+ "dir-key-published ".length());
+ dirKeyPublished = parseFormat.parse(dirKeyPublishedTime).
+ getTime();
+ } else if (line.startsWith("r ")) {
+ String[] parts = line.split(" ");
+ if (parts.length < 9) {
+ this.logger.log(Level.WARNING, "Could not parse r line '"
+ + line + "' in descriptor. Skipping.");
+ break;
+ }
+ String publishedTime = parts[4] + " " + parts[5];
+ String relayIdentity = Hex.encodeHexString(
+ Base64.decodeBase64(parts[2] + "=")).
+ toLowerCase();
+ String serverDesc = Hex.encodeHexString(Base64.decodeBase64(
+ parts[3] + "=")).toLowerCase();
+ serverDescriptors.add(publishedTime + "," + relayIdentity
+ + "," + serverDesc);
+ hashedRelayIdentities.add(DigestUtils.shaHex(
+ Base64.decodeBase64(parts[2] + "=")).
+ toUpperCase());
+ }
+ }
+ if (isConsensus) {
+ if (this.rdd != null) {
+ this.rdd.haveParsedConsensus(validAfterTime, dirSources,
+ serverDescriptors);
+ }
+ if (this.aw != null) {
+ this.aw.storeConsensus(data, validAfter);
+ }
+ } else {
+ if (this.aw != null || this.rdd != null) {
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "network-status-version ";
+ String sigToken = "directory-signature ";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken);
+ if (start >= 0 && sig >= 0 && sig > start) {
+ sig += sigToken.length();
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ String digest = DigestUtils.shaHex(forDigest).toUpperCase();
+ if (this.aw != null) {
+ this.aw.storeVote(data, validAfter, dirSource, digest);
+ }
+ if (this.rdd != null) {
+ this.rdd.haveParsedVote(validAfterTime, fingerprint,
+ serverDescriptors);
+ }
+ }
+ if (certificateString != null) {
+ if (this.aw != null) {
+ this.aw.storeCertificate(certificateString.getBytes(),
+ dirSource, dirKeyPublished);
+ }
+ }
+ }
+ }
+ } else if (line.startsWith("router ")) {
+ String publishedTime = null, extraInfoDigest = null,
+ relayIdentifier = null;
+ long published = -1L;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("published ")) {
+ publishedTime = line.substring("published ".length());
+ published = parseFormat.parse(publishedTime).getTime();
+ } else if (line.startsWith("opt fingerprint") ||
+ line.startsWith("fingerprint")) {
+ relayIdentifier = line.substring(line.startsWith("opt ") ?
+ "opt fingerprint".length() : "fingerprint".length()).
+ replaceAll(" ", "").toLowerCase();
+ } else if (line.startsWith("opt extra-info-digest ") ||
+ line.startsWith("extra-info-digest ")) {
+ extraInfoDigest = line.startsWith("opt ") ?
+ line.split(" ")[2].toLowerCase() :
+ line.split(" ")[1].toLowerCase();
+ }
+ }
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "router ";
+ String sigToken = "\nrouter-signature\n";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ String digest = null;
+ if (start >= 0 || sig >= 0 || sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ digest = DigestUtils.shaHex(forDigest);
+ }
+ if (this.aw != null && digest != null) {
+ this.aw.storeServerDescriptor(data, digest, published);
+ }
+ if (this.rdd != null && digest != null) {
+ this.rdd.haveParsedServerDescriptor(publishedTime,
+ relayIdentifier, digest, extraInfoDigest);
+ }
+ } else if (line.startsWith("extra-info ")) {
+ String publishedTime = null, relayIdentifier = line.split(" ")[2];
+ long published = -1L;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("published ")) {
+ publishedTime = line.substring("published ".length());
+ published = parseFormat.parse(publishedTime).getTime();
+ }
+ }
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ String digest = null;
+ int start = ascii.indexOf(startToken);
+ if (start > 0) {
+ /* Do not confuse "extra-info " in "@type extra-info 1.0" with
+ * "extra-info 0000...". TODO This is a hack that should be
+ * solved by using metrics-lib some day. */
+ start = ascii.indexOf("\n" + startToken);
+ if (start > 0) {
+ start++;
+ }
+ }
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ if (start >= 0 || sig >= 0 || sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ digest = DigestUtils.shaHex(forDigest);
+ }
+ if (this.aw != null && digest != null) {
+ this.aw.storeExtraInfoDescriptor(data, digest, published);
+ }
+ if (this.rdd != null && digest != null) {
+ this.rdd.haveParsedExtraInfoDescriptor(publishedTime,
+ relayIdentifier.toLowerCase(), digest);
+ }
+ }
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not parse descriptor. "
+ + "Skipping.", e);
+ } catch (ParseException e) {
+ this.logger.log(Level.WARNING, "Could not parse descriptor. "
+ + "Skipping.", e);
+ }
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
new file mode 100644
index 0000000..1ac593a
--- /dev/null
+++ b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
@@ -0,0 +1,573 @@
+/* Copyright 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db.torperf;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.text.SimpleDateFormat;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/* Download possibly truncated Torperf .data and .extradata files from
+ * configured sources, append them to the files we already have, and merge
+ * the two files into the .tpf format. */
+public class TorperfDownloader {
+
+ private File torperfOutputDirectory = null;
+ private SortedMap<String, String> torperfSources = null;
+ private List<String> torperfFilesLines = null;
+ private Logger logger = null;
+ private SimpleDateFormat dateFormat;
+
+ public TorperfDownloader(File torperfOutputDirectory,
+ SortedMap<String, String> torperfSources,
+ List<String> torperfFilesLines) {
+ if (torperfOutputDirectory == null) {
+ throw new IllegalArgumentException();
+ }
+ this.torperfOutputDirectory = torperfOutputDirectory;
+ this.torperfSources = torperfSources;
+ this.torperfFilesLines = torperfFilesLines;
+ if (!this.torperfOutputDirectory.exists()) {
+ this.torperfOutputDirectory.mkdirs();
+ }
+ this.logger = Logger.getLogger(TorperfDownloader.class.getName());
+ this.dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+ this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.readLastMergedTimestamps();
+ for (String torperfFilesLine : this.torperfFilesLines) {
+ this.downloadAndMergeFiles(torperfFilesLine);
+ }
+ this.writeLastMergedTimestamps();
+ }
+
+ private File torperfLastMergedFile =
+ new File("stats/torperf-last-merged");
+ SortedMap<String, String> lastMergedTimestamps =
+ new TreeMap<String, String>();
+ private void readLastMergedTimestamps() {
+ if (!this.torperfLastMergedFile.exists()) {
+ return;
+ }
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.torperfLastMergedFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(" ");
+ String fileName = null, timestamp = null;
+ if (parts.length == 2) {
+ try {
+ Double.parseDouble(parts[1]);
+ fileName = parts[0];
+ timestamp = parts[1];
+ } catch (NumberFormatException e) {
+ /* Handle below. */
+ }
+ }
+ if (fileName == null || timestamp == null) {
+ this.logger.log(Level.WARNING, "Invalid line '" + line + "' in "
+ + this.torperfLastMergedFile.getAbsolutePath() + ". "
+ + "Ignoring past history of merging .data and .extradata "
+ + "files.");
+ this.lastMergedTimestamps.clear();
+ break;
+ }
+ this.lastMergedTimestamps.put(fileName, timestamp);
+ }
+ br.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Error while reading '"
+ + this.torperfLastMergedFile.getAbsolutePath() + ". Ignoring "
+ + "past history of merging .data and .extradata files.");
+ this.lastMergedTimestamps.clear();
+ }
+ }
+
+ private void writeLastMergedTimestamps() {
+ try {
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.torperfLastMergedFile));
+ for (Map.Entry<String, String> e :
+ this.lastMergedTimestamps.entrySet()) {
+ String fileName = e.getKey();
+ String timestamp = e.getValue();
+ bw.write(fileName + " " + timestamp + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Error while writing '"
+ + this.torperfLastMergedFile.getAbsolutePath() + ". This may "
+ + "result in ignoring history of merging .data and .extradata "
+ + "files in the next execution.", e);
+ }
+ }
+
+ private void downloadAndMergeFiles(String torperfFilesLine) {
+ String[] parts = torperfFilesLine.split(" ");
+ String sourceName = parts[1];
+ int fileSize = -1;
+ try {
+ fileSize = Integer.parseInt(parts[2]);
+ } catch (NumberFormatException e) {
+ this.logger.log(Level.WARNING, "Could not parse file size in "
+ + "TorperfFiles configuration line '" + torperfFilesLine
+ + "'.");
+ return;
+ }
+
+ /* Download and append the .data file. */
+ String dataFileName = parts[3];
+ String sourceBaseUrl = torperfSources.get(sourceName);
+ String dataUrl = sourceBaseUrl + dataFileName;
+ String dataOutputFileName = sourceName + "-" + dataFileName;
+ File dataOutputFile = new File(torperfOutputDirectory,
+ dataOutputFileName);
+ boolean downloadedDataFile = this.downloadAndAppendFile(dataUrl,
+ dataOutputFile, true);
+
+ /* Download and append the .extradata file. */
+ String extradataFileName = parts[4];
+ String extradataUrl = sourceBaseUrl + extradataFileName;
+ String extradataOutputFileName = sourceName + "-" + extradataFileName;
+ File extradataOutputFile = new File(torperfOutputDirectory,
+ extradataOutputFileName);
+ boolean downloadedExtradataFile = this.downloadAndAppendFile(
+ extradataUrl, extradataOutputFile, false);
+
+ /* Merge both files into .tpf format. */
+ if (!downloadedDataFile && !downloadedExtradataFile) {
+ return;
+ }
+ String skipUntil = null;
+ if (this.lastMergedTimestamps.containsKey(dataOutputFileName)) {
+ skipUntil = this.lastMergedTimestamps.get(dataOutputFileName);
+ }
+ try {
+ skipUntil = this.mergeFiles(dataOutputFile, extradataOutputFile,
+ sourceName, fileSize, skipUntil);
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed merging " + dataOutputFile
+ + " and " + extradataOutputFile + ".", e);
+ }
+ if (skipUntil != null) {
+ this.lastMergedTimestamps.put(dataOutputFileName, skipUntil);
+ }
+ }
+
+ private boolean downloadAndAppendFile(String url, File outputFile,
+ boolean isDataFile) {
+
+ /* Read an existing output file to determine which line will be the
+ * first to append to it. */
+ String lastTimestampLine = null;
+ int linesAfterLastTimestampLine = 0;
+ if (outputFile.exists() && outputFile.lastModified() >
+ System.currentTimeMillis() - 330L * 60L * 1000L) {
+ return false;
+ } else if (outputFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ outputFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (isDataFile || line.contains(" LAUNCH")) {
+ lastTimestampLine = line;
+ linesAfterLastTimestampLine = 0;
+ } else {
+ linesAfterLastTimestampLine++;
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed reading '"
+ + outputFile.getAbsolutePath() + "' to determine the first "
+ + "line to append to it.", e);
+ return false;
+ }
+ }
+ try {
+ this.logger.fine("Downloading " + (isDataFile ? ".data" :
+ ".extradata") + " file from '" + url + "' and merging it into "
+ + "'" + outputFile.getAbsolutePath() + "'.");
+ URL u = new URL(url);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ huc.getInputStream()));
+ String line;
+ BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile,
+ true));
+ boolean copyLines = lastTimestampLine == null;
+ while ((line = br.readLine()) != null) {
+ if (copyLines && linesAfterLastTimestampLine == 0) {
+ if (isDataFile || line.contains(" LAUNCH")) {
+ lastTimestampLine = line;
+ }
+ bw.write(line + "\n");
+ } else if (copyLines && linesAfterLastTimestampLine > 0) {
+ linesAfterLastTimestampLine--;
+ } else if (line.equals(lastTimestampLine)) {
+ copyLines = true;
+ }
+ }
+ bw.close();
+ br.close();
+ if (!copyLines) {
+ this.logger.warning("The last timestamp line in '"
+ + outputFile.getAbsolutePath() + "' is not contained in the "
+ + "new file downloaded from '" + url + "'. Cannot append "
+ + "new lines without possibly leaving a gap. Skipping.");
+ return false;
+ }
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed downloading and/or merging '"
+ + url + "'.", e);
+ return false;
+ }
+ if (lastTimestampLine == null) {
+ this.logger.warning("'" + outputFile.getAbsolutePath()
+ + "' doesn't contain any timestamp lines. Unable to check "
+ + "whether that file is stale or not.");
+ } else {
+ long lastTimestampMillis = -1L;
+ if (isDataFile) {
+ lastTimestampMillis = Long.parseLong(lastTimestampLine.substring(
+ 0, lastTimestampLine.indexOf(" "))) * 1000L;
+ } else {
+ lastTimestampMillis = Long.parseLong(lastTimestampLine.substring(
+ lastTimestampLine.indexOf(" LAUNCH=") + " LAUNCH=".length(),
+ lastTimestampLine.indexOf(".",
+ lastTimestampLine.indexOf(" LAUNCH=")))) * 1000L;
+ }
+ if (lastTimestampMillis < System.currentTimeMillis()
+ - 330L * 60L * 1000L) {
+ this.logger.warning("The last timestamp in '"
+ + outputFile.getAbsolutePath() + "' is more than 5:30 hours "
+ + "old: " + lastTimestampMillis);
+ }
+ }
+ return true;
+ }
+
+ private String mergeFiles(File dataFile, File extradataFile,
+ String source, int fileSize, String skipUntil) throws IOException {
+ SortedMap<String, String> config = new TreeMap<String, String>();
+ config.put("SOURCE", source);
+ config.put("FILESIZE", String.valueOf(fileSize));
+ if (!dataFile.exists() || !extradataFile.exists()) {
+ this.logger.warning("File " + dataFile.getAbsolutePath() + " or "
+ + extradataFile.getAbsolutePath() + " is missing.");
+ return null;
+ }
+ this.logger.fine("Merging " + dataFile.getAbsolutePath() + " and "
+ + extradataFile.getAbsolutePath() + " into .tpf format.");
+ BufferedReader brD = new BufferedReader(new FileReader(dataFile)),
+ brE = new BufferedReader(new FileReader(extradataFile));
+ String lineD = brD.readLine(), lineE = brE.readLine();
+ int d = 1, e = 1;
+ String maxDataComplete = null, maxUsedAt = null;
+ while (lineD != null) {
+
+ /* Parse .data line. Every valid .data line will go into the .tpf
+ * format, either with additional information from the .extradata
+ * file or without it. */
+ if (lineD.isEmpty()) {
+ this.logger.finer("Skipping empty line " + dataFile.getName()
+ + ":" + d++ + ".");
+ lineD = brD.readLine();
+ continue;
+ }
+ SortedMap<String, String> data = this.parseDataLine(lineD);
+ if (data == null) {
+ this.logger.finer("Skipping illegal line " + dataFile.getName()
+ + ":" + d++ + " '" + lineD + "'.");
+ lineD = brD.readLine();
+ continue;
+ }
+ String dataComplete = data.get("DATACOMPLETE");
+ double dataCompleteSeconds = Double.parseDouble(dataComplete);
+ if (skipUntil != null && dataComplete.compareTo(skipUntil) < 0) {
+ this.logger.finer("Skipping " + dataFile.getName() + ":"
+ + d++ + " which we already processed before.");
+ lineD = brD.readLine();
+ continue;
+ }
+ maxDataComplete = dataComplete;
+
+ /* Parse .extradata line if available and try to find the one that
+ * matches the .data line. */
+ SortedMap<String, String> extradata = null;
+ while (lineE != null) {
+ if (lineE.isEmpty()) {
+ this.logger.finer("Skipping " + extradataFile.getName() + ":"
+ + e++ + " which is empty.");
+ lineE = brE.readLine();
+ continue;
+ }
+ if (lineE.startsWith("BUILDTIMEOUT_SET ")) {
+ this.logger.finer("Skipping " + extradataFile.getName() + ":"
+ + e++ + " which is a BUILDTIMEOUT_SET line.");
+ lineE = brE.readLine();
+ continue;
+ } else if (lineE.startsWith("ok ") ||
+ lineE.startsWith("error ")) {
+ this.logger.finer("Skipping " + extradataFile.getName() + ":"
+ + e++ + " which is in the old format.");
+ lineE = brE.readLine();
+ continue;
+ }
+ extradata = this.parseExtradataLine(lineE);
+ if (extradata == null) {
+ this.logger.finer("Skipping Illegal line "
+ + extradataFile.getName() + ":" + e++ + " '" + lineE
+ + "'.");
+ lineE = brE.readLine();
+ continue;
+ }
+ if (!extradata.containsKey("USED_AT")) {
+ this.logger.finer("Skipping " + extradataFile.getName() + ":"
+ + e++ + " which doesn't contain a USED_AT element.");
+ lineE = brE.readLine();
+ continue;
+ }
+ String usedAt = extradata.get("USED_AT");
+ double usedAtSeconds = Double.parseDouble(usedAt);
+ if (skipUntil != null && usedAt.compareTo(skipUntil) < 0) {
+ this.logger.finer("Skipping " + extradataFile.getName() + ":"
+ + e++ + " which we already processed before.");
+ lineE = brE.readLine();
+ continue;
+ }
+ maxUsedAt = usedAt;
+ if (Math.abs(usedAtSeconds - dataCompleteSeconds) <= 1.0) {
+ this.logger.fine("Merging " + extradataFile.getName() + ":"
+ + e++ + " into the current .data line.");
+ lineE = brE.readLine();
+ break;
+ } else if (usedAtSeconds > dataCompleteSeconds) {
+ this.logger.finer("Comparing " + extradataFile.getName()
+ + " to the next .data line.");
+ extradata = null;
+ break;
+ } else {
+ this.logger.finer("Skipping " + extradataFile.getName() + ":"
+ + e++ + " which is too old to be merged with "
+ + dataFile.getName() + ":" + d + ".");
+ lineE = brE.readLine();
+ continue;
+ }
+ }
+
+ /* Write output line to .tpf file. */
+ SortedMap<String, String> keysAndValues =
+ new TreeMap<String, String>();
+ if (extradata != null) {
+ keysAndValues.putAll(extradata);
+ }
+ keysAndValues.putAll(data);
+ keysAndValues.putAll(config);
+ this.logger.fine("Writing " + dataFile.getName() + ":" + d++ + ".");
+ lineD = brD.readLine();
+ try {
+ this.writeTpfLine(source, fileSize, keysAndValues);
+ } catch (IOException ex) {
+ this.logger.log(Level.WARNING, "Error writing output line. "
+ + "Aborting to merge " + dataFile.getName() + " and "
+ + extradataFile.getName() + ".", e);
+ break;
+ }
+ }
+ brD.close();
+ brE.close();
+ this.writeCachedTpfLines();
+ if (maxDataComplete == null) {
+ return maxUsedAt;
+ } else if (maxUsedAt == null) {
+ return maxDataComplete;
+ } else if (maxDataComplete.compareTo(maxUsedAt) > 0) {
+ return maxUsedAt;
+ } else {
+ return maxDataComplete;
+ }
+ }
+
+ private SortedMap<Integer, String> dataTimestamps;
+ private SortedMap<String, String> parseDataLine(String line) {
+ String[] parts = line.trim().split(" ");
+ if (line.length() == 0 || parts.length < 20) {
+ return null;
+ }
+ if (this.dataTimestamps == null) {
+ this.dataTimestamps = new TreeMap<Integer, String>();
+ this.dataTimestamps.put(0, "START");
+ this.dataTimestamps.put(2, "SOCKET");
+ this.dataTimestamps.put(4, "CONNECT");
+ this.dataTimestamps.put(6, "NEGOTIATE");
+ this.dataTimestamps.put(8, "REQUEST");
+ this.dataTimestamps.put(10, "RESPONSE");
+ this.dataTimestamps.put(12, "DATAREQUEST");
+ this.dataTimestamps.put(14, "DATARESPONSE");
+ this.dataTimestamps.put(16, "DATACOMPLETE");
+ this.dataTimestamps.put(21, "DATAPERC10");
+ this.dataTimestamps.put(23, "DATAPERC20");
+ this.dataTimestamps.put(25, "DATAPERC30");
+ this.dataTimestamps.put(27, "DATAPERC40");
+ this.dataTimestamps.put(29, "DATAPERC50");
+ this.dataTimestamps.put(31, "DATAPERC60");
+ this.dataTimestamps.put(33, "DATAPERC70");
+ this.dataTimestamps.put(35, "DATAPERC80");
+ this.dataTimestamps.put(37, "DATAPERC90");
+ }
+ SortedMap<String, String> data = new TreeMap<String, String>();
+ try {
+ for (Map.Entry<Integer, String> e : this.dataTimestamps.entrySet()) {
+ int i = e.getKey();
+ if (parts.length > i + 1) {
+ String key = e.getValue();
+ String value = String.format("%s.%02d", parts[i],
+ Integer.parseInt(parts[i + 1]) / 10000);
+ data.put(key, value);
+ }
+ }
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ data.put("WRITEBYTES", parts[18]);
+ data.put("READBYTES", parts[19]);
+ if (parts.length >= 21) {
+ data.put("DIDTIMEOUT", parts[20]);
+ }
+ return data;
+ }
+
+ private SortedMap<String, String> parseExtradataLine(String line) {
+ String[] parts = line.split(" ");
+ SortedMap<String, String> extradata = new TreeMap<String, String>();
+ String previousKey = null;
+ for (String part : parts) {
+ String[] keyAndValue = part.split("=", -1);
+ if (keyAndValue.length == 2) {
+ String key = keyAndValue[0];
+ previousKey = key;
+ String value = keyAndValue[1];
+ if (value.contains(".") && value.lastIndexOf(".") ==
+ value.length() - 2) {
+ /* Make sure that all floats have two trailing digits. */
+ value += "0";
+ }
+ extradata.put(key, value);
+ } else if (keyAndValue.length == 1 && previousKey != null) {
+ String value = keyAndValue[0];
+ if (previousKey.equals("STREAM_FAIL_REASONS") &&
+ (value.equals("MISC") || value.equals("EXITPOLICY") ||
+ value.equals("RESOURCELIMIT") ||
+ value.equals("RESOLVEFAILED"))) {
+ extradata.put(previousKey, extradata.get(previousKey) + ":"
+ + value);
+ } else {
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+ return extradata;
+ }
+
+ private String cachedSource;
+ private int cachedFileSize;
+ private String cachedStartDate;
+ private SortedMap<String, String> cachedTpfLines;
+ private void writeTpfLine(String source, int fileSize,
+ SortedMap<String, String> keysAndValues) throws IOException {
+ StringBuilder sb = new StringBuilder();
+ int written = 0;
+ for (Map.Entry<String, String> keyAndValue :
+ keysAndValues.entrySet()) {
+ String key = keyAndValue.getKey();
+ String value = keyAndValue.getValue();
+ sb.append((written++ > 0 ? " " : "") + key + "=" + value);
+ }
+ String line = sb.toString();
+ String startString = keysAndValues.get("START");
+ long startMillis = Long.parseLong(startString.substring(0,
+ startString.indexOf("."))) * 1000L;
+ String startDate = dateFormat.format(startMillis);
+ if (this.cachedTpfLines == null || !source.equals(this.cachedSource) ||
+ fileSize != this.cachedFileSize ||
+ !startDate.equals(this.cachedStartDate)) {
+ this.writeCachedTpfLines();
+ this.readTpfLinesToCache(source, fileSize, startDate);
+ }
+ if (!this.cachedTpfLines.containsKey(startString) ||
+ line.length() > this.cachedTpfLines.get(startString).length()) {
+ this.cachedTpfLines.put(startString, line);
+ }
+ }
+
+ private void readTpfLinesToCache(String source, int fileSize,
+ String startDate) throws IOException {
+ this.cachedTpfLines = new TreeMap<String, String>();
+ this.cachedSource = source;
+ this.cachedFileSize = fileSize;
+ this.cachedStartDate = startDate;
+ File tpfFile = new File(torperfOutputDirectory,
+ startDate.replaceAll("-", "/") + "/"
+ + source + "-" + String.valueOf(fileSize) + "-" + startDate
+ + ".tpf");
+ if (!tpfFile.exists()) {
+ return;
+ }
+ BufferedReader br = new BufferedReader(new FileReader(tpfFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("@type ")) {
+ continue;
+ }
+ if (line.contains("START=")) {
+ String startString = line.substring(line.indexOf("START=")
+ + "START=".length()).split(" ")[0];
+ this.cachedTpfLines.put(startString, line);
+ }
+ }
+ br.close();
+ }
+
+ private void writeCachedTpfLines() throws IOException {
+ if (this.cachedSource == null || this.cachedFileSize == 0 ||
+ this.cachedStartDate == null || this.cachedTpfLines == null) {
+ return;
+ }
+ File tpfFile = new File(torperfOutputDirectory,
+ this.cachedStartDate.replaceAll("-", "/")
+ + "/" + this.cachedSource + "-"
+ + String.valueOf(this.cachedFileSize) + "-"
+ + this.cachedStartDate + ".tpf");
+ tpfFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(tpfFile));
+ bw.write("@type torperf 1.0\n");
+ for (String line : this.cachedTpfLines.values()) {
+ bw.write(line + "\n");
+ }
+ bw.close();
+ this.cachedSource = null;
+ this.cachedFileSize = 0;
+ this.cachedStartDate = null;
+ this.cachedTpfLines = null;
+ }
+}
+
diff --git a/test/org/torproject/ernie/db/ArchiveReaderTest.java b/test/org/torproject/ernie/db/ArchiveReaderTest.java
deleted file mode 100644
index cb3dea7..0000000
--- a/test/org/torproject/ernie/db/ArchiveReaderTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Copyright 2011 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.File;
-
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class ArchiveReaderTest {
-
- private File tempArchivesDirectory;
- private File tempStatsDirectory;
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Before
- public void createTempDirectories() {
- this.tempArchivesDirectory = folder.newFolder("sanitized-bridges");
- this.tempStatsDirectory = folder.newFolder("stats");
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testRelayDescriptorParserNull() {
- new ArchiveReader(null, this.tempArchivesDirectory,
- this.tempStatsDirectory, false);
- }
-}
-
diff --git a/test/org/torproject/ernie/db/ArchiveWriterTest.java b/test/org/torproject/ernie/db/ArchiveWriterTest.java
deleted file mode 100644
index 2a18e74..0000000
--- a/test/org/torproject/ernie/db/ArchiveWriterTest.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/* Copyright 2011 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class ArchiveWriterTest {
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Test(expected = IllegalArgumentException.class)
- public void testArchivesDirectoryNull() {
- new ArchiveWriter(null);
- }
-}
-
diff --git a/test/org/torproject/ernie/db/BridgeSnapshotReaderTest.java b/test/org/torproject/ernie/db/BridgeSnapshotReaderTest.java
deleted file mode 100644
index f1ad03b..0000000
--- a/test/org/torproject/ernie/db/BridgeSnapshotReaderTest.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/* Copyright 2011 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.File;
-
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class BridgeSnapshotReaderTest {
-
- private File tempBridgeDirectoriesDirectory;
- private File tempStatsDirectory;
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Before
- public void createTempDirectories() {
- this.tempBridgeDirectoriesDirectory = folder.newFolder("bridges");
- this.tempStatsDirectory = folder.newFolder("stats");
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testBridgeDescriptorParserNull() {
- new BridgeSnapshotReader(null, this.tempBridgeDirectoriesDirectory,
- this.tempStatsDirectory);
- }
-}
-
diff --git a/test/org/torproject/ernie/db/CachedRelayDescriptorReaderTest.java b/test/org/torproject/ernie/db/CachedRelayDescriptorReaderTest.java
deleted file mode 100644
index f101249..0000000
--- a/test/org/torproject/ernie/db/CachedRelayDescriptorReaderTest.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/* Copyright 2011 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.File;
-import java.util.ArrayList;
-
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class CachedRelayDescriptorReaderTest {
-
- private File tempStatsDirectory;
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Before
- public void createTempDirectories() {
- this.tempStatsDirectory = folder.newFolder("stats");
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testRelayDescriptorParserNull() {
- new CachedRelayDescriptorReader(null, new ArrayList<String>(),
- this.tempStatsDirectory);
- }
-}
-
diff --git a/test/org/torproject/ernie/db/SanitizedBridgesWriterTest.java b/test/org/torproject/ernie/db/SanitizedBridgesWriterTest.java
deleted file mode 100644
index f8b4cdd..0000000
--- a/test/org/torproject/ernie/db/SanitizedBridgesWriterTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/* Copyright 2011 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.File;
-
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-public class SanitizedBridgesWriterTest {
-
- private File tempSanitizedBridgesDirectory;
- private File tempStatsDirectory;
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Before
- public void createTempDirectories() {
- this.tempSanitizedBridgesDirectory =
- folder.newFolder("sanitized-bridges");
- this.tempStatsDirectory = folder.newFolder("stats");
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testSanitizedBridgesDirectoryNull() {
- new SanitizedBridgesWriter(null, this.tempStatsDirectory, false, -1L);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testStatsDirectoryNull() {
- new SanitizedBridgesWriter(this.tempSanitizedBridgesDirectory, null,
- false, -1L);
- }
-}
-
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits