[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [metrics-db/master] Remove bridge statistics from metrics-db.
commit 85dc1ad844373bb88fb67bc94e32269990438fcb
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed Mar 2 20:32:09 2011 +0100
Remove bridge statistics from metrics-db.
---
config.template | 22 -
.../ernie/db/BridgeDescriptorParser.java | 158 ++-----
.../ernie/db/BridgeStatsFileHandler.java | 519 --------------------
src/org/torproject/ernie/db/Configuration.java | 59 +--
.../ernie/db/ConsensusStatsFileHandler.java | 282 -----------
src/org/torproject/ernie/db/Main.java | 39 +--
.../torproject/ernie/db/RelayDescriptorParser.java | 14 +-
.../ernie/db/SanitizedBridgesReader.java | 114 -----
.../ernie/test/SanitizedBridgesReaderTest.java | 33 --
9 files changed, 42 insertions(+), 1198 deletions(-)
diff --git a/config.template b/config.template
index 7385f7b..3477801 100644
--- a/config.template
+++ b/config.template
@@ -26,18 +26,6 @@
## download missing relay descriptors from
#DownloadFromDirectoryAuthorities 86.59.21.38,194.109.206.212,212.112.245.170
#
-## Import sanitized bridges from disk, if available
-#ImportSanitizedBridges 0
-#
-## Relative path to directory to import sanitized bridges from
-#SanitizedBridgesDirectory bridges/
-#
-## Keep a history of imported sanitized bridge descriptors. This history
-## can be useful when importing from a changing data source to avoid
-## importing descriptors more than once, but it can be confusing to users
-## who don't know about it.
-#KeepSanitizedBridgesImportHistory 0
-#
## Import bridge snapshots from disk, if available
#ImportBridgeSnapshots 0
#
@@ -75,10 +63,6 @@
## JDBC string for relay descriptor database
#RelayDescriptorDatabaseJDBC jdbc:postgresql://localhost/tordir?user=ernie&password=password
#
-## Write statistics about the current consensus and votes to the
-## website
-#WriteConsensusHealth 0
-#
## Write sanitized bridges to disk
#WriteSanitizedBridges 0
#
@@ -93,10 +77,4 @@
#
## Relative path to directory to write sanitized bridges to
#SanitizedBridgesWriteDirectory sanitized-bridges/
-#
-## Write consensus stats to disk
-#WriteConsensusStats 0
-#
-## Write bridge stats to disk
-#WriteBridgeStats 0
diff --git a/src/org/torproject/ernie/db/BridgeDescriptorParser.java b/src/org/torproject/ernie/db/BridgeDescriptorParser.java
index e2d8bd0..9453088 100644
--- a/src/org/torproject/ernie/db/BridgeDescriptorParser.java
+++ b/src/org/torproject/ernie/db/BridgeDescriptorParser.java
@@ -9,14 +9,9 @@ import java.util.logging.*;
import org.apache.commons.codec.digest.*;
public class BridgeDescriptorParser {
- private ConsensusStatsFileHandler csfh;
- private BridgeStatsFileHandler bsfh;
private SanitizedBridgesWriter sbw;
private Logger logger;
- public BridgeDescriptorParser(ConsensusStatsFileHandler csfh,
- BridgeStatsFileHandler bsfh, SanitizedBridgesWriter sbw) {
- this.csfh = csfh;
- this.bsfh = bsfh;
+ public BridgeDescriptorParser(SanitizedBridgesWriter sbw) {
this.sbw = sbw;
this.logger =
Logger.getLogger(BridgeDescriptorParser.class.getName());
@@ -25,139 +20,46 @@ public class BridgeDescriptorParser {
try {
BufferedReader br = new BufferedReader(new StringReader(
new String(allData, "US-ASCII")));
- SimpleDateFormat timeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- timeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String hashedIdentity = null, platformLine = null,
- publishedLine = null, geoipStartTimeLine = null,
- bridgeStatsEndLine = null;
boolean skip = false;
- String line = null;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("r ")) {
- if (this.sbw != null) {
- if (sanitized) {
- this.sbw.storeSanitizedNetworkStatus(allData, dateTime);
- } else {
- this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime);
- }
+ String line = br.readLine();
+ if (line != null) {
+ return;
+ } else if (line.startsWith("r ")) {
+ if (this.sbw != null) {
+ if (sanitized) {
+ this.sbw.storeSanitizedNetworkStatus(allData, dateTime);
+ } else {
+ this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime);
}
- int runningBridges = 0;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("s ") && line.contains(" Running")) {
- runningBridges++;
- }
- }
- if (this.csfh != null) {
- this.csfh.addBridgeConsensusResults(dateTime, runningBridges);
- }
- } else if (line.startsWith("router ")) {
- if (this.sbw != null) {
- if (sanitized) {
- this.sbw.storeSanitizedServerDescriptor(allData);
- } else {
- this.sbw.sanitizeAndStoreServerDescriptor(allData);
- }
- }
- } else if (line.startsWith("extra-info ")) {
- if (this.sbw != null) {
- if (sanitized) {
- this.sbw.storeSanitizedExtraInfoDescriptor(allData);
- } else {
- this.sbw.sanitizeAndStoreExtraInfoDescriptor(allData);
- }
- }
- hashedIdentity = sanitized ? line.split(" ")[2]
- : DigestUtils.shaHex(line.split(" ")[2]).toUpperCase();
- if (this.bsfh != null) {
- skip = this.bsfh.isKnownRelay(hashedIdentity);
- }
- } else if (!skip && line.startsWith("platform ")) {
- platformLine = line;
- } else if (!skip && line.startsWith("published ")) {
- publishedLine = line;
- } else if (line.startsWith("opt fingerprint") ||
- line.startsWith("fingerprint")) {
- String identity = line.substring(line.startsWith("opt ") ?
- "opt fingerprint".length() : "fingerprint".length()).
- replaceAll(" ", "").toLowerCase();
- hashedIdentity = sanitized ? identity
- : DigestUtils.shaHex(identity).toUpperCase();
- } else if (!skip && line.startsWith("geoip-start-time ")) {
- geoipStartTimeLine = line;
- } else if (!skip && line.startsWith("geoip-client-origins")
- && line.split(" ").length > 1) {
- if (publishedLine == null ||
- geoipStartTimeLine == null) {
- this.logger.warning("Either published line or "
- + "geoip-start-time line is not present in "
- + (sanitized ? "sanitized" : "non-sanitized")
- + " bridge descriptors from " + dateTime + ".");
- break;
- }
- long published = timeFormat.parse(publishedLine.
- substring("published ".length())).getTime();
- long started = timeFormat.parse(geoipStartTimeLine.
- substring("geoip-start-time ".length())).getTime();
- long seconds = (published - started) / 1000L;
- double allUsers = 0.0D;
- Map<String, String> obs = new HashMap<String, String>();
- String[] parts = line.split(" ")[1].split(",");
- for (String p : parts) {
- String country = p.substring(0, 2);
- double users = ((double) Long.parseLong(p.substring(3)) - 4L)
- * 86400.0D / ((double) seconds);
- allUsers += users;
- obs.put(country, String.format("%.2f", users));
- }
- obs.put("zy", String.format("%.2f", allUsers));
- String date = publishedLine.split(" ")[1];
- String time = publishedLine.split(" ")[2];
- if (this.bsfh != null) {
- this.bsfh.addObs(hashedIdentity, date, time, obs);
- }
- } else if (!skip && line.startsWith("bridge-stats-end ")) {
- bridgeStatsEndLine = line;
- } else if (!skip && line.startsWith("bridge-ips")
- && line.split(" ").length > 1) {
- if (bridgeStatsEndLine == null) {
- this.logger.warning("bridge-ips line without preceding "
- + "bridge-stats-end line in "
- + (sanitized ? "sanitized" : "non-sanitized")
- + " bridge descriptor.");
- break;
+ }
+ int runningBridges = 0;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("s ") && line.contains(" Running")) {
+ runningBridges++;
}
- double allUsers = 0.0D;
- Map<String, String> obs = new HashMap<String, String>();
- String[] parts = line.split(" ")[1].split(",");
- for (String p : parts) {
- String country = p.substring(0, 2);
- double users = (double) Long.parseLong(p.substring(3)) - 4L;
- allUsers += users;
- obs.put(country, String.format("%.2f", users));
+ }
+ } else if (line.startsWith("router ")) {
+ if (this.sbw != null) {
+ if (sanitized) {
+ this.sbw.storeSanitizedServerDescriptor(allData);
+ } else {
+ this.sbw.sanitizeAndStoreServerDescriptor(allData);
}
- obs.put("zy", String.format("%.2f", allUsers));
- String date = bridgeStatsEndLine.split(" ")[1];
- String time = bridgeStatsEndLine.split(" ")[2];
- if (this.bsfh != null) {
- this.bsfh.addObs(hashedIdentity, date, time, obs);
+ }
+ } else if (line.startsWith("extra-info ")) {
+ if (this.sbw != null) {
+ if (sanitized) {
+ this.sbw.storeSanitizedExtraInfoDescriptor(allData);
+ } else {
+ this.sbw.sanitizeAndStoreExtraInfoDescriptor(allData);
}
}
}
- if (this.bsfh != null && platformLine != null &&
- platformLine.startsWith("platform Tor 0.2.2")) {
- String date = publishedLine.split(" ")[1];
- String time = publishedLine.split(" ")[2];
- this.bsfh.addZeroTwoTwoDescriptor(hashedIdentity, date, time);
- }
} catch (IOException e) {
this.logger.log(Level.WARNING, "Could not parse bridge descriptor.",
e);
return;
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not parse bridge descriptor.",
- e);
- return;
}
}
}
+
diff --git a/src/org/torproject/ernie/db/BridgeStatsFileHandler.java b/src/org/torproject/ernie/db/BridgeStatsFileHandler.java
deleted file mode 100644
index 8a417b5..0000000
--- a/src/org/torproject/ernie/db/BridgeStatsFileHandler.java
+++ /dev/null
@@ -1,519 +0,0 @@
-/* Copyright 2010 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.*;
-import java.sql.*;
-import java.text.*;
-import java.util.*;
-import java.util.logging.*;
-
-/**
- * Determines estimates of bridge users per country and day from the
- * extra-info descriptors that bridges publish. In a first step, the
- * number of unique IP addresses that bridges see are normalized to a
- * 24-hour period. In the next step, all bridges are excluded that have
- * been running as a relay. Finally, observations are summed up and
- * written to <code>stats/bridge-stats</code>.
- */
-public class BridgeStatsFileHandler {
-
- /**
- * Two-letter country codes of known countries.
- */
- private SortedSet<String> countries;
-
- /**
- * Intermediate results file containing bridge user numbers by country
- * as seen by single bridges, normalized to 24-hour periods.
- */
- private File bridgeStatsRawFile;
-
- /**
- * Bridge user numbers by country as seen by single bridges on a given
- * day. Map keys are bridge and date written as "bridge,date", map
- * values are lines as read from <code>stats/bridge-stats-raw</code>.
- */
- private SortedMap<String, Map<String, String>> bridgeUsersRaw;
-
- /**
- * Helper file containing the hashed relay identities of all known
- * relays. These hashes are compared to the bridge identity hashes to
- * exclude bridges that have been known as relays from the statistics.
- */
- private File hashedRelayIdentitiesFile;
-
- /**
- * Known hashed relay identities used to exclude bridges that have been
- * running as relays.
- */
- private SortedSet<String> hashedRelays;
-
- /**
- * Helper file containing extra-info descriptors published by 0.2.2.x
- * bridges. If these descriptors contain geoip-stats, they are not
- * included in the results, because stats are very likely broken.
- */
- private File zeroTwoTwoDescriptorsFile;
-
- /**
- * Extra-info descriptors published by 0.2.2.x bridges. If these
- * descriptors contain geoip-stats, they are not included in the
- * results, because stats are very likely broken.
- */
- private SortedSet<String> zeroTwoTwoDescriptors;
-
- /**
- * Final results file containing the number of bridge users per country
- * and day. This file is not read in during initialization, but
- * overwritten at the end of the execution.
- */
- private File bridgeStatsFile;
-
- /**
- * Logger for this class.
- */
- private Logger logger;
-
- /* Database connection string. */
- private String connectionURL = null;
-
- /**
- * Initializes this class, including reading in intermediate results
- * files <code>stats/bridge-stats-raw</code> and
- * <code>stats/hashed-relay-identities</code>.
- */
- public BridgeStatsFileHandler(String connectionURL) {
-
- /* Initialize set of known countries. */
- this.countries = new TreeSet<String>();
- this.countries.add("zy");
-
- /* Initialize local data structures to hold results. */
- this.bridgeUsersRaw = new TreeMap<String, Map<String, String>>();
- this.hashedRelays = new TreeSet<String>();
- this.zeroTwoTwoDescriptors = new TreeSet<String>();
-
- /* Initialize file names for intermediate and final results. */
- this.bridgeStatsRawFile = new File("stats/bridge-stats-raw");
- this.bridgeStatsFile = new File("stats/bridge-stats");
- this.hashedRelayIdentitiesFile = new File(
- "stats/hashed-relay-identities");
- this.zeroTwoTwoDescriptorsFile = new File(
- "stats/v022-bridge-descriptors");
-
- /* Initialize database connection string. */
- this.connectionURL = connectionURL;
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(
- BridgeStatsFileHandler.class.getName());
-
- /* Read in bridge user numbers by country as seen by single bridges,
- * normalized to 24-hour periods. */
- if (this.bridgeStatsRawFile.exists()) {
- try {
- this.logger.fine("Reading file "
- + this.bridgeStatsRawFile.getAbsolutePath() + "...");
- BufferedReader br = new BufferedReader(new FileReader(
- this.bridgeStatsRawFile));
- String line = br.readLine();
- if (line != null) {
- /* The first line should contain headers that we need to parse
- * in order to learn what countries we were interested in when
- * writing this file. */
- if (!line.startsWith("bridge,date,time,")) {
- this.logger.warning("Incorrect first line '" + line + "' in "
- + this.bridgeStatsRawFile.getAbsolutePath() + "! This line "
- + "should contain headers! Aborting to read in this "
- + "file!");
- } else {
- String[] headers = line.split(",");
- for (int i = 3; i < headers.length; i++) {
- if (!headers[i].equals("all")) {
- this.countries.add(headers[i]);
- }
- }
- /* Read in the rest of the file. */
- while ((line = br.readLine()) != null) {
- String[] parts = line.split(",");
- if (parts.length != headers.length) {
- this.logger.warning("Corrupt line '" + line + "' in file "
- + this.bridgeStatsRawFile.getAbsolutePath()
- + "! Aborting to read this file!");
- break;
- }
- String hashedBridgeIdentity = parts[0];
- String date = parts[1];
- String time = parts[2];
- SortedMap<String, String> obs =
- new TreeMap<String, String>();
- for (int i = 3; i < parts.length; i++) {
- if (parts[i].equals("NA")) {
- continue;
- }
- if (headers[i].equals("all")) {
- obs.put("zy", parts[i]);
- } else {
- obs.put(headers[i], parts[i]);
- }
- }
- this.addObs(hashedBridgeIdentity, date, time, obs);
- }
- }
- }
- br.close();
- this.logger.fine("Finished reading file "
- + this.bridgeStatsRawFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to read file "
- + this.bridgeStatsRawFile.getAbsolutePath() + "!", e);
- }
- }
-
- /* Read in known hashed relay identities used to exclude bridges that
- * have been running as relays. */
- if (this.hashedRelayIdentitiesFile.exists()) {
- try {
- this.logger.fine("Reading file "
- + this.hashedRelayIdentitiesFile.getAbsolutePath() + "...");
- BufferedReader br = new BufferedReader(new FileReader(
- this.hashedRelayIdentitiesFile));
- String line = null;
- /* Read in all lines from the file and memorize them. */
- while ((line = br.readLine()) != null) {
- this.hashedRelays.add(line);
- }
- br.close();
- this.logger.fine("Finished reading file "
- + this.hashedRelayIdentitiesFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to read file "
- + this.hashedRelayIdentitiesFile.getAbsolutePath() + "!", e);
- }
- }
-
- /* Read in known extra-info descriptors published by 0.2.2.x
- * bridges. */
- if (this.zeroTwoTwoDescriptorsFile.exists()) {
- try {
- this.logger.fine("Reading file "
- + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "...");
- BufferedReader br = new BufferedReader(new FileReader(
- this.zeroTwoTwoDescriptorsFile));
- String line = null;
- /* Read in all lines from the file and memorize them. */
- while ((line = br.readLine()) != null) {
- this.zeroTwoTwoDescriptors.add(line);
- }
- br.close();
- this.logger.fine("Finished reading file "
- + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to read file "
- + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "!", e);
- }
- }
- }
-
- /**
- * Adds a hashed relay identity string to the list of bridges that we
- * are going to ignore in the future. If we counted user numbers from
- * bridges that have been running as relays, our numbers would be far
- * higher than what we think is correct.
- */
- public void addHashedRelay(String hashedRelayIdentity) {
- if (!this.hashedRelays.contains(hashedRelayIdentity)) {
- this.logger.finer("Adding new hashed relay identity: "
- + hashedRelayIdentity);
- this.hashedRelays.add(hashedRelayIdentity);
- }
- }
-
- /**
- * Adds an extra-info descriptor identifier published by an 0.2.2.x
- * bridges. If this extra-info descriptor contains geoip-stats, they are
- * not included in the results, because stats are very likely broken.
- */
- public void addZeroTwoTwoDescriptor(String hashedBridgeIdentity,
- String date, String time) {
- String value = hashedBridgeIdentity.toUpperCase() + "," + date + ","
- + time;
- if (!this.zeroTwoTwoDescriptors.contains(value)) {
- this.logger.finer("Adding new bridge 0.2.2.x extra-info "
- + "descriptor: " + value);
- this.zeroTwoTwoDescriptors.add(value);
- }
- }
-
- /**
- * Returns whether the given fingerprint is a known hashed relay
- * identity. <code>BridgeDescriptorParser</code> uses this information
- * to decide whether to continue parsing a bridge extra-descriptor
- * descriptor or not.
- */
- public boolean isKnownRelay(String hashedBridgeIdentity) {
- return this.hashedRelays.contains(hashedBridgeIdentity);
- }
-
- /**
- * Adds bridge user numbers by country as seen by a single bridge on a
- * given date and time. Bridges can publish statistics on unique IP
- * addresses multiple times a day, but we only want to include one
- * observation per day. If we already have an observation from the given
- * bridge and day, we keep the one with the later publication time and
- * discard the other one.
- */
- public void addObs(String hashedIdentity, String date, String time,
- Map<String, String> obs) {
- for (String country : obs.keySet()) {
- this.countries.add(country);
- }
- String shortKey = hashedIdentity + "," + date;
- String longKey = shortKey + "," + time;
- SortedMap<String, Map<String, String>> tailMap =
- this.bridgeUsersRaw.tailMap(shortKey);
- String nextKey = tailMap.isEmpty() ? null : tailMap.firstKey();
- if (nextKey == null || !nextKey.startsWith(shortKey)) {
- this.logger.finer("Adding new bridge user numbers for key "
- + longKey);
- this.bridgeUsersRaw.put(longKey, obs);
- } else if (longKey.compareTo(nextKey) > 0) {
- this.logger.finer("Replacing existing bridge user numbers (" +
- nextKey + " with new numbers: " + longKey);
- this.bridgeUsersRaw.put(longKey, obs);
- } else {
- this.logger.finer("Not replacing existing bridge user numbers (" +
- nextKey + " with new numbers (" + longKey + ").");
- }
- }
-
- /**
- * Writes the list of hashed relay identities and bridge user numbers as
- * observed by single bridges to disk, aggregates per-day statistics for
- * all bridges, and writes those to disk, too.
- */
- public void writeFiles() {
-
- /* Write hashed relay identities to disk. */
- try {
- this.logger.fine("Writing file "
- + this.hashedRelayIdentitiesFile.getAbsolutePath() + "...");
- this.hashedRelayIdentitiesFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.hashedRelayIdentitiesFile));
- for (String hashedRelay : this.hashedRelays) {
- bw.append(hashedRelay + "\n");
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.hashedRelayIdentitiesFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to write "
- + this.hashedRelayIdentitiesFile.getAbsolutePath() + "!", e);
- }
-
- /* Write bridge extra-info descriptor identifiers to disk. */
- try {
- this.logger.fine("Writing file "
- + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "...");
- this.zeroTwoTwoDescriptorsFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.zeroTwoTwoDescriptorsFile));
- for (String descriptorIdentifier : this.zeroTwoTwoDescriptors) {
- bw.append(descriptorIdentifier + "\n");
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to write "
- + this.zeroTwoTwoDescriptorsFile.getAbsolutePath() + "!", e);
- }
-
- /* Write observations made by single bridges to disk. */
- try {
- this.logger.fine("Writing file "
- + this.bridgeStatsRawFile.getAbsolutePath() + "...");
- this.bridgeStatsRawFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.bridgeStatsRawFile));
- bw.append("bridge,date,time");
- for (String c : this.countries) {
- if (c.equals("zy")) {
- bw.append(",all");
- } else {
- bw.append("," + c);
- }
- }
- bw.append("\n");
- for (Map.Entry<String, Map<String, String>> e :
- this.bridgeUsersRaw.entrySet()) {
- String longKey = e.getKey();
- String[] parts = longKey.split(",");
- String hashedBridgeIdentity = parts[0];
- String date = parts[1];
- String time = parts[2];
- if (!this.hashedRelays.contains(hashedBridgeIdentity) &&
- !this.zeroTwoTwoDescriptors.contains(longKey)) {
- Map<String, String> obs = e.getValue();
- StringBuilder sb = new StringBuilder(longKey);
- for (String c : this.countries) {
- sb.append("," + (obs.containsKey(c) &&
- !obs.get(c).startsWith("-") ? obs.get(c) : "NA"));
- }
- String line = sb.toString();
- bw.append(line + "\n");
- }
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.bridgeStatsRawFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to write "
- + this.bridgeStatsRawFile.getAbsolutePath() + "!", e);
- }
-
- /* Aggregate per-day statistics. */
- SortedMap<String, double[]> bridgeUsersPerDay =
- new TreeMap<String, double[]>();
- for (Map.Entry<String, Map<String, String>> e :
- this.bridgeUsersRaw.entrySet()) {
- String longKey = e.getKey();
- String[] parts = longKey.split(",");
- String hashedBridgeIdentity = parts[0];
- String date = parts[1];
- String time = parts[2];
- if (!this.hashedRelays.contains(hashedBridgeIdentity) &&
- !this.zeroTwoTwoDescriptors.contains(longKey)) {
- double[] users = bridgeUsersPerDay.get(date);
- Map<String, String> obs = e.getValue();
- if (users == null) {
- users = new double[this.countries.size()];
- bridgeUsersPerDay.put(date, users);
- }
- int i = 0;
- for (String c : this.countries) {
- if (obs.containsKey(c) && !obs.get(c).startsWith("-")) {
- users[i] += Double.parseDouble(obs.get(c));
- }
- i++;
- }
- }
- }
-
- /* Write final results of bridge users per day and country to
- * <code>stats/bridge-stats</code>. */
- try {
- this.logger.fine("Writing file "
- + this.bridgeStatsRawFile.getAbsolutePath() + "...");
- this.bridgeStatsFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.bridgeStatsFile));
- bw.append("date");
- for (String c : this.countries) {
- if (c.equals("zy")) {
- bw.append(",all");
- } else {
- bw.append("," + c);
- }
- }
- bw.append("\n");
-
- /* Write current observation. */
- for (Map.Entry<String, double[]> e : bridgeUsersPerDay.entrySet()) {
- String date = e.getKey();
- bw.append(date);
- double[] users = e.getValue();
- for (int i = 0; i < users.length; i++) {
- bw.append("," + String.format("%.2f", users[i]));
- }
- bw.append("\n");
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.bridgeStatsFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to write "
- + this.bridgeStatsFile.getAbsolutePath() + "!", e);
- }
-
- /* Add daily bridge users to database. */
- if (connectionURL != null) {
- try {
- List<String> countryList = new ArrayList<String>();
- for (String c : this.countries) {
- countryList.add(c);
- }
- Map<String, Integer> insertRows = new HashMap<String, Integer>(),
- updateRows = new HashMap<String, Integer>();
- for (Map.Entry<String, double[]> e :
- bridgeUsersPerDay.entrySet()) {
- String date = e.getKey();
- double[] users = e.getValue();
- for (int i = 0; i < users.length; i++) {
- int usersInt = (int) users[i];
- if (usersInt < 1) {
- continue;
- }
- String country = countryList.get(i);
- String key = date + "," + country;
- insertRows.put(key, usersInt);
- }
- }
- Connection conn = DriverManager.getConnection(connectionURL);
- conn.setAutoCommit(false);
- Statement statement = conn.createStatement();
- ResultSet rs = statement.executeQuery(
- "SELECT date, country, users FROM bridge_stats");
- while (rs.next()) {
- String date = rs.getDate(1).toString();
- String country = rs.getString(2);
- String key = date + "," + country;
- if (insertRows.containsKey(key)) {
- int insertRow = insertRows.remove(key);
- int oldUsers = rs.getInt(3);
- if (oldUsers != insertRow) {
- updateRows.put(key, insertRow);
- }
- }
- }
- rs.close();
- PreparedStatement psU = conn.prepareStatement(
- "UPDATE bridge_stats SET users = ? "
- + "WHERE date = ? AND country = ?");
- for (Map.Entry<String, Integer> e : updateRows.entrySet()) {
- String[] keyParts = e.getKey().split(",");
- java.sql.Date date = java.sql.Date.valueOf(keyParts[0]);
- String country = keyParts[1];
- int users = e.getValue();
- psU.clearParameters();
- psU.setInt(1, users);
- psU.setDate(2, date);
- psU.setString(3, country);
- psU.executeUpdate();
- }
- PreparedStatement psI = conn.prepareStatement(
- "INSERT INTO bridge_stats (users, date, country) "
- + "VALUES (?, ?, ?)");
- for (Map.Entry<String, Integer> e : insertRows.entrySet()) {
- String[] keyParts = e.getKey().split(",");
- java.sql.Date date = java.sql.Date.valueOf(keyParts[0]);
- String country = keyParts[1];
- int users = e.getValue();
- psI.clearParameters();
- psI.setInt(1, users);
- psI.setDate(2, date);
- psI.setString(3, country);
- psI.executeUpdate();
- }
- conn.commit();
- conn.close();
- } catch (SQLException e) {
- logger.log(Level.WARNING, "Failed to add daily bridge users to "
- + "database.", e);
- }
- }
- }
-}
-
diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java
index b8f8f39..3554e23 100644
--- a/src/org/torproject/ernie/db/Configuration.java
+++ b/src/org/torproject/ernie/db/Configuration.java
@@ -13,8 +13,6 @@ import java.util.logging.*;
* configuration.
*/
public class Configuration {
- private boolean writeConsensusStats = false;
- private boolean writeBridgeStats = false;
private boolean writeDirectoryArchives = false;
private String directoryArchivesOutputDirectory = "directory-archive/";
private boolean importCachedRelayDescriptors = false;
@@ -30,9 +28,6 @@ public class Configuration {
private boolean replaceIPAddressesWithHashes = false;
private long limitBridgeDescriptorMappings = -1L;
private String sanitizedBridgesWriteDirectory = "sanitized-bridges/";
- private boolean importSanitizedBridges = false;
- private String sanitizedBridgesDirectory = "bridges/";
- private boolean keepSanitizedBridgesImportHistory = false;
private boolean importBridgeSnapshots = false;
private String bridgeSnapshotsDirectory = "bridge-directories/";
private boolean importWriteTorperfStats = false;
@@ -67,12 +62,6 @@ public class Configuration {
while ((line = br.readLine()) != null) {
if (line.startsWith("#") || line.length() < 1) {
continue;
- } else if (line.startsWith("WriteConsensusStats")) {
- this.writeConsensusStats = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("WriteBridgeStats")) {
- this.writeBridgeStats = Integer.parseInt(
- line.split(" ")[1]) != 0;
} else if (line.startsWith("WriteDirectoryArchives")) {
this.writeDirectoryArchives = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -111,14 +100,6 @@ public class Configuration {
line.split(" ")[1]);
} else if (line.startsWith("SanitizedBridgesWriteDirectory")) {
this.sanitizedBridgesWriteDirectory = line.split(" ")[1];
- } else if (line.startsWith("ImportSanitizedBridges")) {
- this.importSanitizedBridges = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("SanitizedBridgesDirectory")) {
- this.sanitizedBridgesDirectory = line.split(" ")[1];
- } else if (line.startsWith("KeepSanitizedBridgesImportHistory")) {
- this.keepSanitizedBridgesImportHistory = Integer.parseInt(
- line.split(" ")[1]) != 0;
} else if (line.startsWith("ImportBridgeSnapshots")) {
this.importBridgeSnapshots = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -184,13 +165,11 @@ public class Configuration {
/** Make some checks if configuration is valid. */
if (!this.importCachedRelayDescriptors &&
!this.importDirectoryArchives && !this.downloadRelayDescriptors &&
- !this.importSanitizedBridges && !this.importBridgeSnapshots &&
- !this.importWriteTorperfStats &&
+ !this.importBridgeSnapshots && !this.importWriteTorperfStats &&
!this.downloadProcessGetTorStats && !this.downloadExitList &&
!this.writeDirectoryArchives &&
!this.writeAggregateStatsDatabase &&
- !this.writeSanitizedBridges && !this.writeConsensusStats &&
- !this.writeBridgeStats) {
+ !this.writeSanitizedBridges) {
logger.warning("We have not been configured to read data from any "
+ "data source or write data to any data sink. You need to "
+ "edit your config file (" + configFile.getAbsolutePath()
@@ -199,8 +178,7 @@ public class Configuration {
}
if ((this.importCachedRelayDescriptors ||
this.importDirectoryArchives || this.downloadRelayDescriptors) &&
- !(this.writeDirectoryArchives || this.writeConsensusStats ||
- this.writeBridgeStats)) {
+ !this.writeDirectoryArchives) {
logger.warning("We are configured to import/download relay "
+ "descriptors, but we don't have a single data sink to write "
+ "relay descriptors to.");
@@ -212,23 +190,13 @@ public class Configuration {
+ "least one data sink, but we don't have a single data source "
+ "containing relay descriptors.");
}
- if (!(this.importCachedRelayDescriptors ||
- this.importDirectoryArchives || this.downloadRelayDescriptors ||
- this.importSanitizedBridges || this.importBridgeSnapshots) &&
- (this.writeBridgeStats || this.writeConsensusStats)) {
- logger.warning("We are configured to write relay or bridge "
- + "descriptors to at least one data sink, but we have neither "
- + "data sources containing relay nor bridge descriptors.");
- }
- if ((this.importSanitizedBridges || this.importBridgeSnapshots) &&
- !(this.writeSanitizedBridges || this.writeConsensusStats ||
- this.writeBridgeStats || this.writeAggregateStatsDatabase)) {
+ if (this.importBridgeSnapshots && !(this.writeSanitizedBridges ||
+ this.writeAggregateStatsDatabase)) {
logger.warning("We are configured to import/download bridge "
+ "descriptors, but we don't have a single data sink to write "
+ "bridge descriptors to.");
}
- if (!(this.importSanitizedBridges || this.importBridgeSnapshots) &&
- (this.writeSanitizedBridges)) {
+ if (!this.importBridgeSnapshots && this.writeSanitizedBridges) {
logger.warning("We are configured to write bridge descriptor to at "
+ "least one data sink, but we don't have a single data source "
+ "containing bridge descriptors.");
@@ -239,12 +207,6 @@ public class Configuration {
+ "but not to import them into the database.");
}
}
- public boolean getWriteConsensusStats() {
- return this.writeConsensusStats;
- }
- public boolean getWriteBridgeStats() {
- return this.writeBridgeStats;
- }
public boolean getWriteDirectoryArchives() {
return this.writeDirectoryArchives;
}
@@ -284,15 +246,6 @@ public class Configuration {
public String getSanitizedBridgesWriteDirectory() {
return this.sanitizedBridgesWriteDirectory;
}
- public boolean getImportSanitizedBridges() {
- return this.importSanitizedBridges;
- }
- public String getSanitizedBridgesDirectory() {
- return this.sanitizedBridgesDirectory;
- }
- public boolean getKeepSanitizedBridgesImportHistory() {
- return this.keepSanitizedBridgesImportHistory;
- }
public boolean getImportBridgeSnapshots() {
return this.importBridgeSnapshots;
}
diff --git a/src/org/torproject/ernie/db/ConsensusStatsFileHandler.java b/src/org/torproject/ernie/db/ConsensusStatsFileHandler.java
deleted file mode 100644
index 81e4e3a..0000000
--- a/src/org/torproject/ernie/db/ConsensusStatsFileHandler.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/* Copyright 2010 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.*;
-import java.sql.*;
-import java.text.*;
-import java.util.*;
-import java.util.logging.*;
-
-/**
- * Generates statistics on the average number of relays and bridges per
- * day. Accepts parse results from <code>RelayDescriptorParser</code> and
- * <code>BridgeDescriptorParser</code> and stores them in intermediate
- * result files <code>stats/consensus-stats-raw</code> and
- * <code>stats/bridge-consensus-stats-raw</code>. Writes final results to
- * <code>stats/consensus-stats</code> for all days for which at least half
- * of the expected consensuses or statuses are known.
- */
-public class ConsensusStatsFileHandler {
-
- /**
- * Intermediate results file holding the number of running bridges per
- * bridge status.
- */
- private File bridgeConsensusStatsRawFile;
-
- /**
- * Number of running bridges in a given bridge status. Map keys are
- * bridge status times formatted as "yyyy-MM-dd HH:mm:ss", map values
- * are lines as read from <code>stats/bridge-consensus-stats-raw</code>.
- */
- private SortedMap<String, String> bridgesRaw;
-
- /**
- * Average number of running bridges per day. Map keys are dates
- * formatted as "yyyy-MM-dd", map values are the last column as written
- * to <code>stats/consensus-stats</code>.
- */
- private SortedMap<String, String> bridgesPerDay;
-
- /**
- * Logger for this class.
- */
- private Logger logger;
-
- private int bridgeResultsAdded = 0;
-
- /* Database connection string. */
- private String connectionURL = null;
-
- /**
- * Initializes this class, including reading in intermediate results
- * files <code>stats/consensus-stats-raw</code> and
- * <code>stats/bridge-consensus-stats-raw</code> and final results file
- * <code>stats/consensus-stats</code>.
- */
- public ConsensusStatsFileHandler(String connectionURL) {
-
- /* Initialize local data structures to hold intermediate and final
- * results. */
- this.bridgesPerDay = new TreeMap<String, String>();
- this.bridgesRaw = new TreeMap<String, String>();
-
- /* Initialize file names for intermediate and final results files. */
- this.bridgeConsensusStatsRawFile = new File(
- "stats/bridge-consensus-stats-raw");
-
- /* Initialize database connection string. */
- this.connectionURL = connectionURL;
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(
- ConsensusStatsFileHandler.class.getName());
-
- /* Read in number of running bridges per bridge status. */
- if (this.bridgeConsensusStatsRawFile.exists()) {
- try {
- this.logger.fine("Reading file "
- + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "...");
- BufferedReader br = new BufferedReader(new FileReader(
- this.bridgeConsensusStatsRawFile));
- String line = null;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("date")) {
- /* Skip headers. */
- continue;
- }
- String[] parts = line.split(",");
- if (parts.length != 2) {
- this.logger.warning("Corrupt line '" + line + "' in file "
- + this.bridgeConsensusStatsRawFile.getAbsolutePath()
- + "! Aborting to read this file!");
- break;
- }
- String dateTime = parts[0];
- this.bridgesRaw.put(dateTime, line);
- }
- br.close();
- this.logger.fine("Finished reading file "
- + this.bridgeConsensusStatsRawFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to read file "
- + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "!",
- e);
- }
- }
- }
-
- /**
- * Adds the intermediate results of the number of running bridges in a
- * given bridge status to the existing observations.
- */
- public void addBridgeConsensusResults(String published, int running) {
- String line = published + "," + running;
- if (!this.bridgesRaw.containsKey(published)) {
- this.logger.finer("Adding new bridge numbers: " + line);
- this.bridgesRaw.put(published, line);
- this.bridgeResultsAdded++;
- } else if (!line.equals(this.bridgesRaw.get(published))) {
- this.logger.warning("The numbers of running bridges we were just "
- + "given (" + line + ") are different from what we learned "
- + "before (" + this.bridgesRaw.get(published) + ")! "
- + "Overwriting!");
- this.bridgesRaw.put(published, line);
- }
- }
-
- /**
- * Aggregates the raw observations on relay and bridge numbers and
- * writes both raw and aggregate observations to disk.
- */
- public void writeFiles() {
-
- /* Did we learn anything new about average relay or bridge numbers in
- * this run? */
- boolean writeConsensusStats = false;
-
- /* Go through raw observations of numbers of running bridges in bridge
- * statuses, calculate averages per day, and add these averages to
- * final results. */
- if (!this.bridgesRaw.isEmpty()) {
- String tempDate = null;
- int brunning = 0, statuses = 0;
- Iterator<String> it = this.bridgesRaw.values().iterator();
- boolean haveWrittenFinalLine = false;
- while (it.hasNext() || !haveWrittenFinalLine) {
- String next = it.hasNext() ? it.next() : null;
- /* Finished reading a day or even all lines? */
- if (tempDate != null && (next == null
- || !next.substring(0, 10).equals(tempDate))) {
- /* Only write results if we have seen at least half of all
- * statuses. */
- if (statuses >= 24) {
- String line = "," + (brunning / statuses);
- /* Are our results new? */
- if (!this.bridgesPerDay.containsKey(tempDate)) {
- this.logger.finer("Adding new average bridge numbers: "
- + tempDate + line);
- this.bridgesPerDay.put(tempDate, line);
- writeConsensusStats = true;
- } else if (!line.equals(this.bridgesPerDay.get(tempDate))) {
- this.logger.finer("Replacing existing average bridge "
- + "numbers (" + this.bridgesPerDay.get(tempDate)
- + " with new numbers: " + line);
- this.bridgesPerDay.put(tempDate, line);
- writeConsensusStats = true;
- }
- }
- brunning = statuses = 0;
- haveWrittenFinalLine = (next == null);
- }
- /* Sum up number of running bridges. */
- if (next != null) {
- tempDate = next.substring(0, 10);
- statuses++;
- brunning += Integer.parseInt(next.split(",")[1]);
- }
- }
- }
-
- /* Write raw numbers of running bridges to disk. */
- try {
- this.logger.fine("Writing file "
- + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "...");
- this.bridgeConsensusStatsRawFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(
- new FileWriter(this.bridgeConsensusStatsRawFile));
- bw.append("datetime,brunning\n");
- for (String line : this.bridgesRaw.values()) {
- bw.append(line + "\n");
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.bridgeConsensusStatsRawFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to write file "
- + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "!",
- e);
- }
-
- /* Add average number of bridges per day to the database. */
- if (connectionURL != null) {
- try {
- Map<String, String> insertRows = new HashMap<String, String>(),
- updateRows = new HashMap<String, String>();
- insertRows.putAll(this.bridgesPerDay);
- Connection conn = DriverManager.getConnection(connectionURL);
- conn.setAutoCommit(false);
- Statement statement = conn.createStatement();
- ResultSet rs = statement.executeQuery(
- "SELECT date, avg_running FROM bridge_network_size");
- while (rs.next()) {
- String date = rs.getDate(1).toString();
- if (insertRows.containsKey(date)) {
- String insertRow = insertRows.remove(date);
- long newAvgRunning = Long.parseLong(insertRow.substring(1));
- long oldAvgRunning = rs.getLong(2);
- if (newAvgRunning != oldAvgRunning) {
- updateRows.put(date, insertRow);
- }
- }
- }
- rs.close();
- PreparedStatement psU = conn.prepareStatement(
- "UPDATE bridge_network_size SET avg_running = ? "
- + "WHERE date = ?");
- for (Map.Entry<String, String> e : updateRows.entrySet()) {
- java.sql.Date date = java.sql.Date.valueOf(e.getKey());
- long avgRunning = Long.parseLong(e.getValue().substring(1));
- psU.clearParameters();
- psU.setLong(1, avgRunning);
- psU.setDate(2, date);
- psU.executeUpdate();
- }
- PreparedStatement psI = conn.prepareStatement(
- "INSERT INTO bridge_network_size (avg_running, date) "
- + "VALUES (?, ?)");
- for (Map.Entry<String, String> e : insertRows.entrySet()) {
- java.sql.Date date = java.sql.Date.valueOf(e.getKey());
- long avgRunning = Long.parseLong(e.getValue().substring(1));
- psI.clearParameters();
- psI.setLong(1, avgRunning);
- psI.setDate(2, date);
- psI.executeUpdate();
- }
- conn.commit();
- conn.close();
- } catch (SQLException e) {
- logger.log(Level.WARNING, "Failed to add average bridge numbers "
- + "to database.", e);
- }
- }
-
- /* Write stats. */
- StringBuilder dumpStats = new StringBuilder("Finished writing "
- + "statistics on bridge network statuses to disk.\nAdded "
- + this.bridgeResultsAdded + " bridge network status(es) in this "
- + "execution.");
- long now = System.currentTimeMillis();
- SimpleDateFormat dateTimeFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (this.bridgesRaw.isEmpty()) {
- dumpStats.append("\nNo bridge status known yet.");
- } else {
- dumpStats.append("\nLast known bridge status was published "
- + this.bridgesRaw.lastKey() + ".");
- try {
- if (now - 6L * 60L * 60L * 1000L > dateTimeFormat.parse(
- this.bridgesRaw.lastKey()).getTime()) {
- logger.warning("Last known bridge status is more than 6 hours "
- + "old: " + this.bridgesRaw.lastKey());
- }
- } catch (ParseException e) {
- /* Can't parse the timestamp? Whatever. */
- }
- }
- logger.info(dumpStats.toString());
- }
-}
-
diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java
index 4dced2c..2dc0c86 100644
--- a/src/org/torproject/ernie/db/Main.java
+++ b/src/org/torproject/ernie/db/Main.java
@@ -33,12 +33,6 @@ public class Main {
// Define stats directory for temporary files
File statsDirectory = new File("stats");
- // Prepare bridge stats file handler
- BridgeStatsFileHandler bsfh = config.getWriteBridgeStats() ?
- new BridgeStatsFileHandler(
- config.getWriteAggregateStatsDatabase() ?
- config.getRelayDescriptorDatabaseJDBC() : null) : null;
-
// Prepare writing relay descriptor archive to disk
ArchiveWriter aw = config.getWriteDirectoryArchives() ?
new ArchiveWriter(
@@ -46,9 +40,8 @@ public class Main {
// Prepare relay descriptor parser (only if we are writing stats or
// directory archives to disk)
- RelayDescriptorParser rdp = config.getWriteBridgeStats() ||
- config.getWriteDirectoryArchives() ?
- new RelayDescriptorParser(bsfh, aw) : null;
+ RelayDescriptorParser rdp = aw != null ?
+ new RelayDescriptorParser(aw) : null;
// Import/download relay descriptors from the various sources
if (rdp != null) {
@@ -56,7 +49,7 @@ public class Main {
if (config.getDownloadRelayDescriptors()) {
List<String> dirSources =
config.getDownloadFromDirectoryAuthorities();
- boolean downloadCurrentConsensus = aw != null || bsfh != null;
+ boolean downloadCurrentConsensus = aw != null;
boolean downloadCurrentVotes = aw != null;
boolean downloadAllServerDescriptors = aw != null;
boolean downloadAllExtraInfos = aw != null;
@@ -100,12 +93,6 @@ public class Main {
aw = null;
}
- // Prepare consensus stats file handler
- ConsensusStatsFileHandler csfh = config.getWriteConsensusStats() ?
- new ConsensusStatsFileHandler(
- config.getWriteAggregateStatsDatabase() ?
- config.getRelayDescriptorDatabaseJDBC() : null) : null;
-
// Prepare sanitized bridge descriptor writer
SanitizedBridgesWriter sbw = config.getWriteSanitizedBridges() ?
new SanitizedBridgesWriter(
@@ -114,16 +101,10 @@ public class Main {
config.getLimitBridgeDescriptorMappings()) : null;
// Prepare bridge descriptor parser
- BridgeDescriptorParser bdp = config.getWriteConsensusStats() ||
- config.getWriteBridgeStats() || config.getWriteSanitizedBridges()
- ? new BridgeDescriptorParser(csfh, bsfh, sbw) : null;
+ BridgeDescriptorParser bdp = config.getWriteSanitizedBridges()
+ ? new BridgeDescriptorParser(sbw) : null;
// Import bridge descriptors
- if (bdp != null && config.getImportSanitizedBridges()) {
- new SanitizedBridgesReader(bdp,
- new File(config.getSanitizedBridgesDirectory()),
- statsDirectory, config.getKeepSanitizedBridgesImportHistory());
- }
if (bdp != null && config.getImportBridgeSnapshots()) {
new BridgeSnapshotReader(bdp,
new File(config.getBridgeSnapshotsDirectory()),
@@ -136,16 +117,6 @@ public class Main {
sbw = null;
}
- // Write updated stats files to disk
- if (bsfh != null) {
- bsfh.writeFiles();
- bsfh = null;
- }
- if (csfh != null) {
- csfh.writeFiles();
- csfh = null;
- }
-
// Import and process torperf stats
if (config.getImportWriteTorperfStats()) {
new TorperfProcessor(new File(config.getTorperfDirectory()),
diff --git a/src/org/torproject/ernie/db/RelayDescriptorParser.java b/src/org/torproject/ernie/db/RelayDescriptorParser.java
index fcfc0eb..bfd0033 100644
--- a/src/org/torproject/ernie/db/RelayDescriptorParser.java
+++ b/src/org/torproject/ernie/db/RelayDescriptorParser.java
@@ -18,11 +18,6 @@ import org.apache.commons.codec.binary.*;
public class RelayDescriptorParser {
/**
- * Stats file handler that accepts parse results for bridge statistics.
- */
- private BridgeStatsFileHandler bsfh;
-
- /**
* File writer that writes descriptor contents to files in a
* directory-archive directory structure.
*/
@@ -44,9 +39,7 @@ public class RelayDescriptorParser {
/**
* Initializes this class.
*/
- public RelayDescriptorParser(BridgeStatsFileHandler bsfh,
- ArchiveWriter aw) {
- this.bsfh = bsfh;
+ public RelayDescriptorParser(ArchiveWriter aw) {
this.aw = aw;
/* Initialize logger. */
@@ -121,11 +114,6 @@ public class RelayDescriptorParser {
}
}
if (isConsensus) {
- if (this.bsfh != null) {
- for (String hashedRelayIdentity : hashedRelayIdentities) {
- this.bsfh.addHashedRelay(hashedRelayIdentity);
- }
- }
if (this.rdd != null) {
this.rdd.haveParsedConsensus(validAfterTime, dirSources,
serverDescriptors);
diff --git a/src/org/torproject/ernie/db/SanitizedBridgesReader.java b/src/org/torproject/ernie/db/SanitizedBridgesReader.java
deleted file mode 100644
index 2ddf1dd..0000000
--- a/src/org/torproject/ernie/db/SanitizedBridgesReader.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/* Copyright 2010 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.*;
-import java.util.*;
-import java.util.logging.*;
-
-public class SanitizedBridgesReader {
- public SanitizedBridgesReader(BridgeDescriptorParser bdp,
- File bridgesDir, File statsDirectory, boolean keepImportHistory) {
-
- if (bdp == null || bridgesDir == null || statsDirectory == null) {
- throw new IllegalArgumentException();
- }
-
- Logger logger =
- Logger.getLogger(SanitizedBridgesReader.class.getName());
- SortedSet<String> bridgesImportHistory = new TreeSet<String>();
- File bridgesImportHistoryFile =
- new File(statsDirectory, "bridges-import-history");
- if (keepImportHistory && bridgesImportHistoryFile.exists()) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- bridgesImportHistoryFile));
- String line = null;
- while ((line = br.readLine()) != null) {
- bridgesImportHistory.add(line);
- }
- br.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not read in bridge descriptor "
- + "import history file. Skipping.");
- }
- }
- if (bridgesDir.exists()) {
- logger.fine("Importing files in directory " + bridgesDir + "/...");
- Stack<File> filesInInputDir = new Stack<File>();
- filesInInputDir.add(bridgesDir);
- List<File> problems = new ArrayList<File>();
- while (!filesInInputDir.isEmpty()) {
- File pop = filesInInputDir.pop();
- if (pop.isDirectory()) {
- for (File f : pop.listFiles()) {
- filesInInputDir.add(f);
- }
- continue;
- } else if (keepImportHistory && bridgesImportHistory.contains(
- pop.getName())) {
- continue;
- } else {
- try {
- BufferedInputStream bis = new BufferedInputStream(
- new FileInputStream(pop));
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- bis.close();
- byte[] allData = baos.toByteArray();
- String fn = pop.getName();
- // TODO dateTime extraction doesn't work for sanitized network
- // statuses!
- String dateTime = fn.substring(0, 4) + "-" + fn.substring(4, 6)
- + "-" + fn.substring(6, 8) + " " + fn.substring(9, 11)
- + ":" + fn.substring(11, 13) + ":" + fn.substring(13, 15);
- bdp.parse(allData, dateTime, true);
- if (keepImportHistory) {
- bridgesImportHistory.add(pop.getName());
- }
- } catch (IOException e) {
- problems.add(pop);
- if (problems.size() > 3) {
- break;
- }
- }
- }
- }
- if (problems.isEmpty()) {
- logger.fine("Finished importing files in directory " + bridgesDir
- + "/.");
- } else {
- StringBuilder sb = new StringBuilder("Failed importing files in "
- + "directory " + bridgesDir + "/:");
- int printed = 0;
- for (File f : problems) {
- sb.append("\n " + f.getAbsolutePath());
- if (++printed >= 3) {
- sb.append("\n ... more");
- break;
- }
- }
- logger.warning(sb.toString());
- }
- if (keepImportHistory) {
- try {
- bridgesImportHistoryFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- bridgesImportHistoryFile));
- for (String line : bridgesImportHistory) {
- bw.write(line + "\n");
- }
- bw.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not write bridge descriptor "
- + "import history file.");
- }
- }
- }
- }
-}
-
diff --git a/src/org/torproject/ernie/test/SanitizedBridgesReaderTest.java b/src/org/torproject/ernie/test/SanitizedBridgesReaderTest.java
deleted file mode 100644
index dd5f31e..0000000
--- a/src/org/torproject/ernie/test/SanitizedBridgesReaderTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/* Copyright 2011 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.test;
-
-import org.torproject.ernie.db.*;
-
-import java.io.*;
-
-import org.junit.*;
-import org.junit.rules.*;
-import static org.junit.Assert.*;
-
-public class SanitizedBridgesReaderTest {
-
- private File tempSanitizedBridgesDirectory;
- private File tempStatsDirectory;
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Before
- public void createTempDirectories() {
- this.tempSanitizedBridgesDirectory = folder.newFolder("bridges");
- this.tempStatsDirectory = folder.newFolder("stats");
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testBridgeDescriptorParserNull() {
- new SanitizedBridgesReader(null, this.tempSanitizedBridgesDirectory,
- this.tempStatsDirectory, false);
- }
-}
-
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits