[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] [metrics-db/master] Remove unused GeoIPDatabaseManager.
commit bfd398c39b1d484e9ebeaf1be22b55ae1fcff739
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Thu Dec 30 10:08:44 2010 +0100
Remove unused GeoIPDatabaseManager.
We once needed to mantain different GeoIP database versions to resolve
bridge IP addresses to country codes in the sanitizing process. We gave
up on that, because it was too complex. That means we don't need the
GeoIP database manager anymore. If we decide we need it, it'll live on in
Git's history.
---
config.template | 13 -
src/org/torproject/ernie/db/Configuration.java | 27 --
.../torproject/ernie/db/GeoIPDatabaseManager.java | 424 --------------------
3 files changed, 0 insertions(+), 464 deletions(-)
diff --git a/config.template b/config.template
index c275c87..5ba3b74 100644
--- a/config.template
+++ b/config.template
@@ -44,19 +44,6 @@
## Relative path to directory to import bridge descriptor snapshots from
#BridgeSnapshotsDirectory bridge-directories/
#
-## Import local Maxmind GeoIP databases
-#ImportGeoIPDatabases 0
-#
-## Relative path to directory to import GeoIP databases from
-#GeoIPDatabasesDirectory geoipdb/
-#
-## Download (commercial) Maxmind GeoIP database for sanitizing bridge
-## descriptors
-#DownloadGeoIPDatabase 0
-#
-## Maxmind license key for downloading commercial GeoIP databases
-#MaxmindLicenseKey NA
-#
## Import torperf data, if available, and write stats to disk
#ImportWriteTorperfStats 0
#
diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java
index 03b4358..79658ae 100644
--- a/src/org/torproject/ernie/db/Configuration.java
+++ b/src/org/torproject/ernie/db/Configuration.java
@@ -19,7 +19,6 @@ public class Configuration {
private boolean writeDirectoryArchives = false;
private String directoryArchivesOutputDirectory = "directory-archive/";
private boolean importCachedRelayDescriptors = false;
- //this.cachedRelayDescriptorDirectory.add
private List<String> cachedRelayDescriptorsDirectory =
new ArrayList<String>(Arrays.asList("cacheddesc/".split(",")));
private boolean importDirectoryArchives = false;
@@ -47,10 +46,6 @@ public class Configuration {
private String getTorStatsUrl = "http://gettor.torproject.org:8080/";
+ "~gettor/gettor_stats.txt";
private boolean downloadExitList = false;
- private boolean importGeoIPDatabases = false;
- private String geoIPDatabasesDirectory = "geoipdb/";
- private boolean downloadGeoIPDatabase = false;
- private String maxmindLicenseKey = "";
private boolean writeConsensusHealth = false;
public Configuration() {
@@ -169,16 +164,6 @@ public class Configuration {
} else if (line.startsWith("DownloadExitList")) {
this.downloadExitList = Integer.parseInt(
line.split(" ")[1]) != 0;
- } else if (line.startsWith("ImportGeoIPDatabases")) {
- this.importGeoIPDatabases = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("GeoIPDatabasesDirectory")) {
- this.geoIPDatabasesDirectory = line.split(" ")[1];
- } else if (line.startsWith("DownloadGeoIPDatabase")) {
- this.downloadGeoIPDatabase = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("MaxmindLicenseKey")) {
- this.maxmindLicenseKey = line.split(" ")[1];
} else if (line.startsWith("WriteConsensusHealth")) {
this.writeConsensusHealth = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -353,18 +338,6 @@ public class Configuration {
public boolean getDownloadExitList() {
return this.downloadExitList;
}
- public boolean getImportGeoIPDatabases() {
- return this.importGeoIPDatabases;
- }
- public String getGeoIPDatabasesDirectory() {
- return this.geoIPDatabasesDirectory;
- }
- public boolean getDownloadGeoIPDatabase() {
- return this.downloadGeoIPDatabase;
- }
- public String getMaxmindLicenseKey() {
- return this.maxmindLicenseKey;
- }
public boolean getWriteConsensusHealth() {
return this.writeConsensusHealth;
}
diff --git a/src/org/torproject/ernie/db/GeoIPDatabaseManager.java b/src/org/torproject/ernie/db/GeoIPDatabaseManager.java
deleted file mode 100644
index 078f192..0000000
--- a/src/org/torproject/ernie/db/GeoIPDatabaseManager.java
+++ /dev/null
@@ -1,424 +0,0 @@
-/* Copyright 2010 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.*;
-import java.net.*;
-import java.text.*;
-import java.util.*;
-import java.util.logging.*;
-import java.util.zip.*;
-
-/**
- * Maintains multiple versions of GeoIP databases to resolve IP addresses
- * to country codes using the most recent database at a given time.
- * Supports importing CSV-formatted databases from disk and downloading
- * the most recent commercial Maxmind GeoIP database from their server
- * using a license key.
- *
- * 0 databases: all requests answered with ZZ
- * 1 database: all requests answered from that database
- * 2+ databases: requests answered by most recent database at given date
- */
-public class GeoIPDatabaseManager {
-
- /**
- * Database entry of the combined GeoIP database consisting of start IP
- * address, end IP address, and countries of all contained database
- * versions.
- */
- private static class DatabaseEntry {
-
- /**
- * Start IP address.
- */
- long fromIP;
-
- /**
- * End IP address.
- */
- long toIP;
-
- /**
- * Countries of all contained database versions.
- */
- String countries;
- }
-
- /**
- * Mapping from an IP address in decimal form to a database entry.
- */
- private SortedMap<Long, DatabaseEntry> combinedDatabase;
-
- /**
- * Has the combined database been modified from importing database
- * versions from disk?
- */
- private boolean combinedDatabaseModified;
-
- /**
- * File holding the combined GeoIP database.
- */
- private File combinedDatabaseFile;
-
- /**
- * List of dates representing the GeoIP database versions.
- */
- private List<String> allDatabases;
-
- /**
- * Timestamp when we last downloaded the GeoIP database from the Maxmind
- * servers.
- */
- private String lastDownloadedTime;
-
- private String geoipDir;
-
- /**
- * Logger for this class.
- */
- private Logger logger;
-
- private Set<String> unresolvedCountryCodes;
-
- /**
- * Initializes this class by reading in the database versions known so
- * far.
- */
- public GeoIPDatabaseManager(String geoipDir) {
-
- /* Initialize instance variables. */
- this.geoipDir = geoipDir;
- this.combinedDatabaseFile = new File("stats/geoip-database");
- this.combinedDatabase = new TreeMap<Long, DatabaseEntry>();
- this.allDatabases = new ArrayList<String>();
- this.combinedDatabaseModified = false;
- this.unresolvedCountryCodes = new HashSet<String>(Arrays.asList(
- "--,a1,a2,eu,ap".split(",")));
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
-
- /* Read in combined GeoIP database. */
- if (this.combinedDatabaseFile.exists()) {
- try {
- this.logger.fine("Reading in "
- + this.combinedDatabaseFile.getAbsolutePath() + "...");
- BufferedReader br = new BufferedReader(new FileReader(
- this.combinedDatabaseFile));
- String line = null;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("lastDownload")) {
- this.lastDownloadedTime = line.substring("lastDownload ".
- length());
- } else if (line.startsWith("beginIpNum,endIpNum")) {
- String[] parts = line.split(",");
- for (int i = 2; i < parts.length; i++) {
- this.allDatabases.add(parts[i]);
- }
- } else {
- String[] parts = line.split(",");
- DatabaseEntry e = new DatabaseEntry();
- e.fromIP = Long.parseLong(parts[0]);
- e.toIP = Long.parseLong(parts[1]);
- e.countries = line.substring(line.indexOf(",",
- line.indexOf(",") + 1));
- this.combinedDatabase.put(e.fromIP, e);
- }
- }
- br.close();
- this.logger.fine("Finished reading in "
- + this.combinedDatabaseFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to read in "
- + this.combinedDatabaseFile.getAbsolutePath() + "!", e);
- }
- }
- }
-
- /**
- * Downloads today's commercial Maxmind GeoIP database, if such a
- * database exists, and writes it to disk. This method should be called
- * before importing GeoIP databases from disk if the new database should
- * be included in the combined database.
- */
- public void downloadGeoIPDatabase(String licenseKey) {
- if (licenseKey == null || licenseKey.length() < 1) {
- logger.warning("Missing or invalid license key for downloading "
- + "GeoIP database!");
- return;
- }
-
- /* Find out when we tried downloading the last time to avoid making
- * too many download attempts. */
- SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- long now = System.currentTimeMillis();
- if (this.lastDownloadedTime != null) {
- long lastDownloaded = -1;
- try {
- lastDownloaded = dateTimeFormat.parse(this.lastDownloadedTime).
- getTime();
- } catch (ParseException e) {
- logger.log(Level.WARNING, "Could not parse last downloaded "
- + "time '" + this.lastDownloadedTime + "'. Ignoring.");
- }
- if (lastDownloaded + 8L * 60L * 60L * 1000L > now) {
- logger.finer("Last GeoIP database download not more than 8 "
- + "hours in the past. Not downloading.");
- return;
- }
- }
-
- /* Download GeoIP database. */
- try {
- logger.fine("Downloading GeoIP database...");
- this.lastDownloadedTime = dateTimeFormat.format(now);
- SimpleDateFormat urlDateFormat = new SimpleDateFormat("yyyyMMdd");
- urlDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String date = urlDateFormat.format(now);
- String url = "http://www.maxmind.com/app/download_new";
- + "?edition_id=108&date=" + date + "&suffix=zip&license_key="
- + licenseKey;
- URL u = new URL(url);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- int response = huc.getResponseCode();
- if (response != 200) {
- logger.fine("Could not download GeoIP database. Response code "
- + response);
- return;
- }
- BufferedInputStream bis = new BufferedInputStream(
- huc.getInputStream());
- ZipInputStream zis = new ZipInputStream(bis);
- BufferedInputStream bis2 = new BufferedInputStream(zis);
- ZipEntry entry = null;
- while ((entry = zis.getNextEntry()) != null) {
- if (!entry.isDirectory() &&
- entry.getName().endsWith("GeoIP-108.csv")) {
- String filename = geoipDir + "/GeoIP-108_" + date + ".csv";
- OutputStream out = new BufferedOutputStream(
- new FileOutputStream(filename));
- byte[] buffer = new byte[1024];
- int len;
- while ((len = bis2.read(buffer)) >= 0) {
- out.write(buffer, 0, len);
- }
- out.close();
- }
- }
- zis.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not download GeoIP database. "
- + "Exiting.", e);
- return;
- }
- }
-
- /**
- * Imports the GeoIP databases to include them in the combined GeoIP
- * database.
- */
- public void importGeoIPDatabaseFromDisk() {
- File databaseDirectory = new File(this.geoipDir);
- if (!databaseDirectory.exists()) {
- return;
- }
- for (File databaseFile : databaseDirectory.listFiles()) {
- String filename = databaseFile.getName();
- if (!filename.startsWith("GeoIP-108_") ||
- filename.length() != "GeoIP-108_xxxxxxxx.csv".length() ||
- !filename.endsWith(".csv")) {
- continue;
- }
- String date = filename.substring("GeoIP-108_".length(),
- "GeoIP-108_xxxxxxxx".length());
- if (allDatabases.contains(date)) {
- continue;
- }
- this.combinedDatabaseModified = true;
- this.logger.fine("Reading in " + filename);
- String emptyCountryString = "";
- for (int i = 0; i < this.allDatabases.size(); i++) {
- emptyCountryString += ",ZZ";
- }
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- databaseFile));
- String line = null;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("Copyright") ||
- line.startsWith("\"begin")) {
- continue;
- }
- String lineWithoutQuotes = line.replaceAll("\"", "");
- String[] parts = lineWithoutQuotes.split(",");
- lineWithoutQuotes = null; // does this help GC?
- long fromIP = Long.parseLong(parts[2]);
- long toIP = Long.parseLong(parts[3]);
- String countryCode = parts[4];
- SortedMap<Long, DatabaseEntry> submap =
- combinedDatabase.headMap(toIP + 1L);
- if (!submap.headMap(fromIP + 1L).isEmpty()) {
- submap = submap.tailMap(submap.headMap(fromIP + 1L).
- lastKey());
- }
- Set<DatabaseEntry> newEntries = new HashSet<DatabaseEntry>();
- for (DatabaseEntry e : submap.values()) {
- while (fromIP <= toIP && fromIP <= e.toIP &&
- toIP >= e.fromIP) {
- if (fromIP < e.fromIP) {
- // duplicate entry: new entry fromIP-e.fromIP, set fromIP
- // to e.fromIP
- DatabaseEntry e1 = new DatabaseEntry();
- e1.fromIP = fromIP;
- e1.toIP = e.fromIP - 1L;
- e1.countries = e.countries;
- newEntries.add(e1);
- fromIP = e.fromIP;
- } else if (fromIP > e.fromIP) {
- // split off existing entry; don't add yet
- DatabaseEntry e1 = new DatabaseEntry();
- e1.fromIP = e.fromIP;
- e1.toIP = fromIP - 1L;
- e1.countries = e.countries;
- newEntries.add(e1);
- e.fromIP = fromIP;
- newEntries.add(e);
- } else if (toIP < e.toIP) {
- // split and add to first half
- DatabaseEntry e1 = new DatabaseEntry();
- e1.fromIP = toIP + 1L;
- e1.toIP = e.toIP;
- e1.countries = e.countries;
- newEntries.add(e1);
- e.toIP = e1.fromIP - 1L;
- e.countries += "," + countryCode;
- fromIP = toIP + 1L;
- } else if (toIP >= e.toIP) {
- // add to this entry and done, right?
- e.countries += "," + countryCode;
- fromIP = toIP + 1L;
- }
- }
- }
- if (fromIP <= toIP) {
- DatabaseEntry entry = new DatabaseEntry();
- entry.fromIP = fromIP;
- entry.toIP = toIP;
- entry.countries = emptyCountryString + "," + countryCode;
- newEntries.add(entry);
- fromIP = toIP + 1L;
- }
- for (DatabaseEntry e : newEntries) {
- this.combinedDatabase.put(e.fromIP, e);
- }
- }
- this.allDatabases.add(date);
- for (DatabaseEntry e : this.combinedDatabase.values()) {
- if (e.countries.substring(1).split(",").length <
- this.allDatabases.size()) {
- e.countries += ",ZZ";
- }
- }
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not import GeoIP database "
- + "from file " + databaseFile.getAbsolutePath()
- + ". This might leave us with an inconsistent state!");
- }
- }
- }
-
- public void writeCombinedDatabase() {
- if (!combinedDatabaseModified) {
- return;
- }
- try {
- this.logger.fine("Writing "
- + this.combinedDatabaseFile.getAbsolutePath() + "...");
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.combinedDatabaseFile));
- bw.write("lastDownload " + this.lastDownloadedTime + "\n");
- bw.write("beginIpNum,endIpNum");
- for (String d : allDatabases) {
- bw.write("," + d);
- }
- bw.write("\n");
- for (DatabaseEntry e : this.combinedDatabase.values()) {
- bw.write(e.fromIP + "," + e.toIP + e.countries + "\n");
- }
- bw.close();
- this.logger.fine("Finished writing "
- + this.combinedDatabaseFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to write "
- + this.combinedDatabaseFile.getAbsolutePath() + "!");
- }
- }
-
- public String getCountryForIPOneWeek(String ipAddress, String date) {
- SimpleDateFormat parseFormat = new SimpleDateFormat("yyyy-MM-dd");
- parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- try {
- String dateMinusOneWeek = parseFormat.format(new Date(
- parseFormat.parse(date).getTime() -
- 7L * 24L * 60L * 60L * 1000L));
- return this.getCountryForIP(ipAddress, dateMinusOneWeek);
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not parse date '" + date
- + "'.", e);
- return null;
- }
- }
-
- /**
- * Returns the uppercase two-letter country code that was assigned to
- * <code>ipAddress</code> (in dotted notation) in the most recent
- * commercial Maxmind GeoIP database published before or at
- * <code>date</code> (in the format yyyy-MM-dd).
- */
- public String getCountryForIP(String ipAddress, String date) {
- if (this.allDatabases.isEmpty()) {
- return "ZZ";
- }
- String dateShort = date.substring(0, 4) + date.substring(5, 7)
- + date.substring(8, 10); // TODO put full date in allDatabases
- String dbDate = null;
- if (this.allDatabases.contains(dateShort)) {
- dbDate = dateShort;
- } else {
- SortedSet<String> subset = new TreeSet<String>(this.allDatabases).
- headSet(dateShort);
- if (!subset.isEmpty()) {
- dbDate = subset.last();
- } else {
- dbDate = this.allDatabases.get(0);
- }
- }
- if (dbDate == null || !this.allDatabases.contains(dbDate)) {
- return "ZZ";
- }
- String[] parts = ipAddress.split("\\.");
- long ipNum = Long.parseLong(parts[0]) * 256 * 256 * 256 +
- Long.parseLong(parts[1]) * 256 * 256 +
- Long.parseLong(parts[2]) * 256 + Long.parseLong(parts[3]);
- String countries = null;
- if (this.combinedDatabase.containsKey(ipNum)) {
- countries = this.combinedDatabase.get(ipNum).countries;
- } else if (!this.combinedDatabase.headMap(ipNum).isEmpty()) {
- countries = this.combinedDatabase.get(this.combinedDatabase.headMap(
- ipNum).lastKey()).countries;
- } else {
- return "ZZ";
- }
- String countryCode = countries.substring(1).split(",")[
- this.allDatabases.indexOf(dbDate)];
- if (unresolvedCountryCodes.contains(countryCode)) {
- return "ZZ";
- }
- return countryCode;
- }
-}
From a605c4b5c31df3648501b1e89a624cd3a01633c4 Mon Sep 17 00:00:00 2001
Patch-Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Subject: [metrics-db/master] Remove unused parts of ConsensusStatsFileHandler.
commit a605c4b5c31df3648501b1e89a624cd3a01633c4
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Thu Jan 6 17:19:18 2011 +0100
Remove unused parts of ConsensusStatsFileHandler.
---
.../ernie/db/ConsensusStatsFileHandler.java | 343 ++------------------
.../torproject/ernie/db/RelayDescriptorParser.java | 4 -
2 files changed, 19 insertions(+), 328 deletions(-)
diff --git a/src/org/torproject/ernie/db/ConsensusStatsFileHandler.java b/src/org/torproject/ernie/db/ConsensusStatsFileHandler.java
index 469e239..81e4e3a 100644
--- a/src/org/torproject/ernie/db/ConsensusStatsFileHandler.java
+++ b/src/org/torproject/ernie/db/ConsensusStatsFileHandler.java
@@ -20,27 +20,6 @@ import java.util.logging.*;
public class ConsensusStatsFileHandler {
/**
- * Intermediate results file holding the number of relays with Exit,
- * Fast, Guard, Running, and Stable flags per consensus.
- */
- private File consensusStatsRawFile;
-
- /**
- * Number of relays in a given consensus with Exit, Fast, Guard,
- * Running, and Stable flags set. Map keys are consensus valid-after
- * times formatted as "yyyy-MM-dd HH:mm:ss", map values are lines as
- * read from <code>stats/consensus-stats-raw</code>.
- */
- private SortedMap<String, String> relaysRaw;
-
- /**
- * Modification flag for <code>relaysRaw</code>. This flag is used to
- * decide whether the contents of <code>relaysRaw</code> need to be
- * written to disk during <code>writeFiles</code>.
- */
- private boolean relaysRawModified;
-
- /**
* Intermediate results file holding the number of running bridges per
* bridge status.
*/
@@ -54,28 +33,6 @@ public class ConsensusStatsFileHandler {
private SortedMap<String, String> bridgesRaw;
/**
- * Modification flag for <code>bridgesRaw</code>. This flag is used to
- * decide whether the contents of <code>bridgesRaw</code> need to be
- * written to disk during <code>writeFiles</code>.
- */
- private boolean bridgesRawModified;
-
- /**
- * Final results file holding the average number of relays with Exit,
- * Fast, Guard, Running, and Stable flags set and the number of running
- * bridges per day.
- */
- private File consensusStatsFile;
-
- /**
- * Average number of relays with Exit, Fast, Guard, Running, and Stable
- * flags set per day. Map keys are dates formatted as "yyyy-MM-dd", map
- * values are lines as written to <code>stats/consensus-stats</code>
- * without the last column that contains the number of running bridges.
- */
- private SortedMap<String, String> relaysPerDay;
-
- /**
* Average number of running bridges per day. Map keys are dates
* formatted as "yyyy-MM-dd", map values are the last column as written
* to <code>stats/consensus-stats</code>.
@@ -87,7 +44,7 @@ public class ConsensusStatsFileHandler {
*/
private Logger logger;
- private int relayResultsAdded = 0, bridgeResultsAdded = 0;
+ private int bridgeResultsAdded = 0;
/* Database connection string. */
private String connectionURL = null;
@@ -102,16 +59,12 @@ public class ConsensusStatsFileHandler {
/* Initialize local data structures to hold intermediate and final
* results. */
- this.relaysPerDay = new TreeMap<String, String>();
this.bridgesPerDay = new TreeMap<String, String>();
- this.relaysRaw = new TreeMap<String, String>();
this.bridgesRaw = new TreeMap<String, String>();
/* Initialize file names for intermediate and final results files. */
- this.consensusStatsRawFile = new File("stats/consensus-stats-raw");
this.bridgeConsensusStatsRawFile = new File(
"stats/bridge-consensus-stats-raw");
- this.consensusStatsFile = new File("stats/consensus-stats");
/* Initialize database connection string. */
this.connectionURL = connectionURL;
@@ -120,38 +73,6 @@ public class ConsensusStatsFileHandler {
this.logger = Logger.getLogger(
ConsensusStatsFileHandler.class.getName());
- /* Read in number of relays with flags set per consensus. */
- if (this.consensusStatsRawFile.exists()) {
- try {
- this.logger.fine("Reading file "
- + this.consensusStatsRawFile.getAbsolutePath() + "...");
- BufferedReader br = new BufferedReader(new FileReader(
- this.consensusStatsRawFile));
- String line = null;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("date")) {
- /* Skip headers. */
- continue;
- }
- String[] parts = line.split(",");
- if (parts.length != 6) {
- this.logger.warning("Corrupt line '" + line + "' in file "
- + this.consensusStatsRawFile.getAbsolutePath()
- + "! Aborting to read this file!");
- break;
- }
- String dateTime = parts[0];
- this.relaysRaw.put(dateTime, line);
- }
- br.close();
- this.logger.fine("Finished reading file "
- + this.consensusStatsRawFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to read file "
- + this.consensusStatsRawFile.getAbsolutePath() + "!", e);
- }
- }
-
/* Read in number of running bridges per bridge status. */
if (this.bridgeConsensusStatsRawFile.exists()) {
try {
@@ -184,75 +105,6 @@ public class ConsensusStatsFileHandler {
e);
}
}
-
- /* Read in previous results on average numbers of relays and running
- * bridges per day. */
- if (this.consensusStatsFile.exists()) {
- try {
- this.logger.fine("Reading file "
- + this.consensusStatsFile.getAbsolutePath() + "...");
- BufferedReader br = new BufferedReader(new FileReader(
- this.consensusStatsFile));
- String line = null;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("date")) {
- /* Skip headers. */
- continue;
- }
- String[] parts = line.split(",");
- if (parts.length != 7) {
- this.logger.warning("Corrupt line '" + line + "' in file "
- + this.consensusStatsFile.getAbsolutePath()
- + "! Aborting to read this file!");
- break;
- }
- String date = parts[0];
- /* Split line into relay and bridge part; the relay part ends
- * with the last comma (excluding) and the bridge part starts at
- * that comma (including). */
- String relayPart = line.substring(0, line.lastIndexOf(","));
- String bridgePart = line.substring(line.lastIndexOf(","));
- if (!relayPart.endsWith(",NA,NA,NA,NA,NA")) {
- this.relaysPerDay.put(date, relayPart);
- }
- if (!bridgePart.equals(",NA")) {
- this.bridgesPerDay.put(date, bridgePart);
- }
- }
- br.close();
- this.logger.fine("Finished reading file "
- + this.consensusStatsFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to write file "
- + this.consensusStatsFile.getAbsolutePath() + "!", e);
- }
- }
-
- /* Set modification flags to false. */
- this.relaysRawModified = this.bridgesRawModified = false;
- }
-
- /**
- * Adds the intermediate results of the number of relays with certain
- * flags in a given consensus to the existing observations.
- */
- public void addConsensusResults(String validAfter, int exit, int fast,
- int guard, int running, int stable) {
- String line = validAfter + "," + exit + "," + fast + "," + guard + ","
- + running + "," + stable;
- if (!this.relaysRaw.containsKey(validAfter)) {
- this.logger.finer("Adding new relay numbers: " + line);
- this.relaysRaw.put(validAfter, line);
- this.relaysRawModified = true;
- this.relayResultsAdded++;
- } else if (!line.equals(this.relaysRaw.get(validAfter))) {
- this.logger.warning("The numbers of relays with Exit, Fast, "
- + "Guard, Running, and Stable flag we were just given (" + line
- + ") are different from what we learned before ("
- + this.relaysRaw.get(validAfter) + ")! Overwriting!");
- this.relaysRaw.put(validAfter, line);
- this.relaysRawModified = true;
- }
}
/**
@@ -264,7 +116,6 @@ public class ConsensusStatsFileHandler {
if (!this.bridgesRaw.containsKey(published)) {
this.logger.finer("Adding new bridge numbers: " + line);
this.bridgesRaw.put(published, line);
- this.bridgesRawModified = true;
this.bridgeResultsAdded++;
} else if (!line.equals(this.bridgesRaw.get(published))) {
this.logger.warning("The numbers of running bridges we were just "
@@ -272,7 +123,6 @@ public class ConsensusStatsFileHandler {
+ "before (" + this.bridgesRaw.get(published) + ")! "
+ "Overwriting!");
this.bridgesRaw.put(published, line);
- this.bridgesRawModified = true;
}
}
@@ -286,57 +136,6 @@ public class ConsensusStatsFileHandler {
* this run? */
boolean writeConsensusStats = false;
- /* Go through raw observations of numbers of relays in consensuses,
- * calculate averages per day, and add these averages to final
- * results. */
- if (!this.relaysRaw.isEmpty()) {
- String tempDate = null;
- int exit = 0, fast = 0, guard = 0, running = 0, stable = 0,
- consensuses = 0;
- Iterator<String> it = this.relaysRaw.values().iterator();
- boolean haveWrittenFinalLine = false;
- while (it.hasNext() || !haveWrittenFinalLine) {
- String next = it.hasNext() ? it.next() : null;
- /* Finished reading a day or even all lines? */
- if (tempDate != null && (next == null
- || !next.substring(0, 10).equals(tempDate))) {
- /* Only write results if we have seen at least half of all
- * consensuses. */
- if (consensuses >= 12) {
- String line = tempDate + "," + (exit / consensuses) + ","
- + (fast/ consensuses) + "," + (guard/ consensuses) + ","
- + (running/ consensuses) + "," + (stable/ consensuses);
- /* Are our results new? */
- if (!this.relaysPerDay.containsKey(tempDate)) {
- this.logger.finer("Adding new average relay numbers: "
- + line);
- this.relaysPerDay.put(tempDate, line);
- writeConsensusStats = true;
- } else if (!line.equals(this.relaysPerDay.get(tempDate))) {
- this.logger.finer("Replacing existing average relay numbers "
- + "(" + this.relaysPerDay.get(tempDate) + " with new "
- + "numbers: " + line);
- this.relaysPerDay.put(tempDate, line);
- writeConsensusStats = true;
- }
- }
- exit = fast = guard = running = stable = consensuses = 0;
- haveWrittenFinalLine = (next == null);
- }
- /* Sum up number of relays with given flags. */
- if (next != null) {
- String[] parts = next.split(",");
- tempDate = next.substring(0, 10);
- consensuses++;
- exit += Integer.parseInt(parts[1]);
- fast += Integer.parseInt(parts[2]);
- guard += Integer.parseInt(parts[3]);
- running += Integer.parseInt(parts[4]);
- stable += Integer.parseInt(parts[5]);
- }
- }
- }
-
/* Go through raw observations of numbers of running bridges in bridge
* statuses, calculate averages per day, and add these averages to
* final results. */
@@ -380,109 +179,24 @@ public class ConsensusStatsFileHandler {
}
}
- /* Write raw numbers of relays with flags set to disk. */
- if (this.relaysRawModified) {
- try {
- this.logger.fine("Writing file "
- + this.consensusStatsRawFile.getAbsolutePath() + "...");
- this.consensusStatsRawFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.consensusStatsRawFile));
- bw.append("datetime,exit,fast,guard,running,stable\n");
- for (String line : this.relaysRaw.values()) {
- bw.append(line + "\n");
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.consensusStatsRawFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to write file "
- + this.consensusStatsRawFile.getAbsolutePath() + "!", e);
- }
- } else {
- this.logger.fine("Not writing file "
- + this.consensusStatsRawFile.getAbsolutePath() + ", because "
- + "nothing has changed.");
- }
-
/* Write raw numbers of running bridges to disk. */
- if (this.bridgesRawModified) {
- try {
- this.logger.fine("Writing file "
- + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "...");
- this.bridgeConsensusStatsRawFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(
- new FileWriter(this.bridgeConsensusStatsRawFile));
- bw.append("datetime,brunning\n");
- for (String line : this.bridgesRaw.values()) {
- bw.append(line + "\n");
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.bridgeConsensusStatsRawFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to write file "
- + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "!",
- e);
+ try {
+ this.logger.fine("Writing file "
+ + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "...");
+ this.bridgeConsensusStatsRawFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(
+ new FileWriter(this.bridgeConsensusStatsRawFile));
+ bw.append("datetime,brunning\n");
+ for (String line : this.bridgesRaw.values()) {
+ bw.append(line + "\n");
}
- } else {
- this.logger.fine("Not writing file "
- + this.bridgeConsensusStatsRawFile.getAbsolutePath()
- + ", because nothing has changed.");
- }
-
- /* Write final results of relays with flags set and running bridges
- * to disk. */
- if (writeConsensusStats) {
- try {
- this.logger.fine("Writing file "
- + this.consensusStatsFile.getAbsolutePath() + "...");
- this.consensusStatsFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.consensusStatsFile));
- bw.append("date,exit,fast,guard,running,stable,brunning\n");
- /* Iterate over all days, including those for which we don't have
- * observations for which we add NA's to all columns. */
- SortedSet<String> allDates = new TreeSet<String>();
- allDates.addAll(this.relaysPerDay.keySet());
- allDates.addAll(this.bridgesPerDay.keySet());
- SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
- dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- long firstDateMillis = dateFormat.parse(allDates.first()).
- getTime();
- long lastDateMillis = dateFormat.parse(allDates.last()).getTime();
- long currentDateMillis = firstDateMillis;
- while (currentDateMillis <= lastDateMillis) {
- /* Write observations about relays, bridges, both, or none of
- * them. */
- String date = dateFormat.format(currentDateMillis);
- if (this.relaysPerDay.containsKey(date)) {
- bw.append(this.relaysPerDay.get(date));
- } else {
- bw.append(date + ",NA,NA,NA,NA,NA");
- }
- if (this.bridgesPerDay.containsKey(date)) {
- bw.append(this.bridgesPerDay.get(date) + "\n");
- } else {
- bw.append(",NA\n");
- }
- /* Advance by 1 day. */
- currentDateMillis += 24L * 60L * 60L * 1000L;
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.consensusStatsFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to write file "
- + this.consensusStatsFile.getAbsolutePath() + "!", e);
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Failed to write file "
- + this.consensusStatsFile.getAbsolutePath() + "!", e);
- }
- } else {
- this.logger.fine("Not writing file "
- + this.consensusStatsFile.getAbsolutePath()
- + ", because nothing has changed.");
+ bw.close();
+ this.logger.fine("Finished writing file "
+ + this.bridgeConsensusStatsRawFile.getAbsolutePath() + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed to write file "
+ + this.bridgeConsensusStatsRawFile.getAbsolutePath() + "!",
+ e);
}
/* Add average number of bridges per day to the database. */
@@ -538,34 +252,15 @@ public class ConsensusStatsFileHandler {
}
}
- /* Set modification flags to false again. */
- this.relaysRawModified = this.bridgesRawModified = false;
-
/* Write stats. */
StringBuilder dumpStats = new StringBuilder("Finished writing "
- + "statistics on relay consensuses and bridge statuses to disk.\n"
- + "Added " + this.relayResultsAdded + " relay consensus(es) and "
- + this.bridgeResultsAdded + " bridge status(es) in this "
+ + "statistics on bridge network statuses to disk.\nAdded "
+ + this.bridgeResultsAdded + " bridge network status(es) in this "
+ "execution.");
long now = System.currentTimeMillis();
SimpleDateFormat dateTimeFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (this.relaysRaw.isEmpty()) {
- dumpStats.append("\nNo relay consensus known yet.");
- } else {
- dumpStats.append("\nLast known relay consensus was published "
- + this.relaysRaw.lastKey() + ".");
- try {
- if (now - 6L * 60L * 60L * 1000L > dateTimeFormat.parse(
- this.relaysRaw.lastKey()).getTime()) {
- logger.warning("Last known relay consensus is more than 6 hours "
- + "old: " + this.relaysRaw.lastKey());
- }
- } catch (ParseException e) {
- /* Can't parse the timestamp? Whatever. */
- }
- }
if (this.bridgesRaw.isEmpty()) {
dumpStats.append("\nNo bridge status known yet.");
} else {
diff --git a/src/org/torproject/ernie/db/RelayDescriptorParser.java b/src/org/torproject/ernie/db/RelayDescriptorParser.java
index a1f6cd3..187bddb 100644
--- a/src/org/torproject/ernie/db/RelayDescriptorParser.java
+++ b/src/org/torproject/ernie/db/RelayDescriptorParser.java
@@ -217,10 +217,6 @@ public class RelayDescriptorParser {
this.bsfh.addHashedRelay(hashedRelayIdentity);
}
}
- if (this.csfh != null) {
- this.csfh.addConsensusResults(validAfterTime, exit, fast,
- guard, running, stable);
- }
if (this.rdd != null) {
this.rdd.haveParsedConsensus(validAfterTime, dirSources,
serverDescriptors);
From e6844fab4f8caeaf551bf534c7d36685b727105f Mon Sep 17 00:00:00 2001
Patch-Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Subject: [metrics-db/master] Parse vote documents in a local data directory, too.
commit e6844fab4f8caeaf551bf534c7d36685b727105f
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Thu Jan 13 13:44:01 2011 +0100
Parse vote documents in a local data directory, too.
---
.../ernie/db/CachedRelayDescriptorReader.java | 28 ++++++++++++++++++-
1 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/src/org/torproject/ernie/db/CachedRelayDescriptorReader.java b/src/org/torproject/ernie/db/CachedRelayDescriptorReader.java
index 3cf7d12..a8ccc14 100644
--- a/src/org/torproject/ernie/db/CachedRelayDescriptorReader.java
+++ b/src/org/torproject/ernie/db/CachedRelayDescriptorReader.java
@@ -72,6 +72,30 @@ public class CachedRelayDescriptorReader {
if (rdp != null) {
rdp.parse(allData);
}
+ } else if (f.getName().equals("v3-status-votes")) {
+ int parsedNum = 0;
+ String ascii = new String(allData, "US-ASCII");
+ String startToken = "network-status-version ";
+ int end = ascii.length();
+ int start = ascii.indexOf(startToken);
+ while (start >= 0 && start < end) {
+ int next = ascii.indexOf(startToken, start + 1);
+ if (next < 0) {
+ next = end;
+ }
+ if (start < next) {
+ byte[] rawNetworkStatusBytes = new byte[next - start];
+ System.arraycopy(allData, start, rawNetworkStatusBytes, 0,
+ next - start);
+ if (rdp != null) {
+ rdp.parse(rawNetworkStatusBytes);
+ }
+ parsedNum++;
+ }
+ start = next;
+ }
+ dumpStats.append("\n" + f.getName() + ": " + parsedNum
+ + " votes");
} else if (f.getName().startsWith("cached-descriptors") ||
f.getName().startsWith("cached-extrainfo")) {
String ascii = new String(allData, "US-ASCII");
@@ -107,8 +131,6 @@ public class CachedRelayDescriptorReader {
dumpStats.append("\n" + f.getName() + ": " + parsedNum + " "
+ (f.getName().startsWith("cached-descriptors") ?
"server" : "extra-info") + " descriptors");
- logger.fine("Finished reading "
- + cachedDescDir.getAbsolutePath() + " directory.");
}
} catch (IOException e) {
logger.log(Level.WARNING, "Failed reading "
@@ -118,6 +140,8 @@ public class CachedRelayDescriptorReader {
+ cachedDescDir.getAbsolutePath() + " directory.", e);
}
}
+ logger.fine("Finished reading "
+ + cachedDescDir.getAbsolutePath() + " directory.");
}
logger.info(dumpStats.toString());
}