[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] [ernie/master] Add GeoIP database manager in preparation for sanitizing bridge descriptors.
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Tue, 23 Mar 2010 18:54:04 +0100
Subject: Add GeoIP database manager in preparation for sanitizing bridge descriptors.
Commit: f9562a646295d6873409ebe5570cc028f7fd521d
---
config | 10 +
src/Configuration.java | 20 +++
src/GeoIPDatabaseManager.java | 377 +++++++++++++++++++++++++++++++++++++++++
src/Main.java | 10 +
4 files changed, 417 insertions(+), 0 deletions(-)
create mode 100644 src/GeoIPDatabaseManager.java
diff --git a/config b/config
index 18c5ed6..00f6388 100644
--- a/config
+++ b/config
@@ -80,3 +80,13 @@
## Download exit list and store it to disk
#DownloadExitList 0
+## Import local Maxmind GeoIP databases
+#ImportGeoIPDatabases 1
+
+## Download (commercial) Maxmind GeoIP database for sanitizing bridge
+## descriptors
+#DownloadGeoIPDatabase 0
+
+## Maxmind license key for downloading commercial GeoIP databases
+#MaxmindLicenseKey NA
+
diff --git a/src/Configuration.java b/src/Configuration.java
index cf8c7d1..0412040 100644
--- a/src/Configuration.java
+++ b/src/Configuration.java
@@ -35,6 +35,9 @@ public class Configuration {
private String getTorStatsUrl = "http://gettor.torproject.org:8080/";
+ "~gettor/gettor_stats.txt";
private boolean downloadExitList = false;
+ private boolean importGeoIPDatabases = true;
+ private boolean downloadGeoIPDatabase = false;
+ private String maxmindLicenseKey = "";
public Configuration() {
Logger logger = Logger.getLogger(Configuration.class.getName());
File configFile = new File("config");
@@ -123,6 +126,14 @@ public class Configuration {
} else if (line.startsWith("DownloadExitList")) {
this.downloadExitList = Integer.parseInt(
line.split(" ")[1]) != 0;
+ } else if (line.startsWith("ImportGeoIPDatabases")) {
+ this.importGeoIPDatabases = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadGeoIPDatabase")) {
+ this.downloadGeoIPDatabase = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("MaxmindLicenseKey")) {
+ this.maxmindLicenseKey = line.split(" ")[1];
} else {
logger.severe("Configuration file contains unrecognized "
+ "configuration key in line '" + line + "'! Exiting!");
@@ -208,5 +219,14 @@ public class Configuration {
public boolean getDownloadExitList() {
return this.downloadExitList;
}
+ public boolean getImportGeoIPDatabases() {
+ return this.importGeoIPDatabases;
+ }
+ public boolean getDownloadGeoIPDatabase() {
+ return this.downloadGeoIPDatabase;
+ }
+ public String getMaxmindLicenseKey() {
+ return this.maxmindLicenseKey;
+ }
}
diff --git a/src/GeoIPDatabaseManager.java b/src/GeoIPDatabaseManager.java
new file mode 100644
index 0000000..7438003
--- /dev/null
+++ b/src/GeoIPDatabaseManager.java
@@ -0,0 +1,377 @@
+import java.io.*;
+import java.net.*;
+import java.text.*;
+import java.util.*;
+import java.util.logging.*;
+import java.util.zip.*;
+
+/**
+ * Maintains multiple versions of GeoIP databases to resolve IP addresses
+ * to country codes using the most recent database at a given time.
+ * Supports importing CSV-formatted databases from disk and downloading
+ * the most recent commercial Maxmind GeoIP database from their server
+ * using a license key.
+ */
+public class GeoIPDatabaseManager {
+
+ /**
+ * Database entry of the combined GeoIP database consisting of start IP
+ * address, end IP address, and countries of all contained database
+ * versions.
+ */
+ private static class DatabaseEntry {
+
+ /**
+ * Start IP address.
+ */
+ long fromIP;
+
+ /**
+ * End IP address.
+ */
+ long toIP;
+
+ /**
+ * Countries of all contained database versions.
+ */
+ String countries;
+ }
+
+ /**
+ * Mapping from an IP address in decimal form to a database entry.
+ */
+ private SortedMap<Long, DatabaseEntry> combinedDatabase;
+
+ /**
+ * Has the combined database been modified from importing database
+ * versions from disk?
+ */
+ private boolean combinedDatabaseModified;
+
+ /**
+ * File holding the combined GeoIP database.
+ */
+ private File combinedDatabaseFile;
+
+ /**
+ * List of dates representing the GeoIP database versions.
+ */
+ private List<String> allDatabases;
+
+ /**
+ * Timestamp when we last downloaded the GeoIP database from the Maxmind
+ * servers.
+ */
+ private String lastDownloadedTime;
+
+ /**
+ * Logger for this class.
+ */
+ private Logger logger;
+
+ /**
+ * Initializes this class by reading in the database versions known so
+ * far.
+ */
+ public GeoIPDatabaseManager() {
+
+ /* Initialize instance variables. */
+ this.combinedDatabaseFile = new File("stats/geoip-database");
+ this.combinedDatabase = new TreeMap<Long, DatabaseEntry>();
+ this.allDatabases = new ArrayList<String>();
+ this.combinedDatabaseModified = false;
+
+ /* Initialize logger. */
+ this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
+
+ /* Read in combined GeoIP database. */
+ if (this.combinedDatabaseFile.exists()) {
+ try {
+ this.logger.fine("Reading in "
+ + this.combinedDatabaseFile.getAbsolutePath() + "...");
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.combinedDatabaseFile));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("lastDownload")) {
+ this.lastDownloadedTime = line.substring("lastDownload ".
+ length());
+ } else if (line.startsWith("beginIpNum,endIpNum")) {
+ String[] parts = line.split(",");
+ for (int i = 2; i < parts.length; i++) {
+ this.allDatabases.add(parts[i]);
+ }
+ } else {
+ String[] parts = line.split(",");
+ DatabaseEntry e = new DatabaseEntry();
+ e.fromIP = Long.parseLong(parts[0]);
+ e.toIP = Long.parseLong(parts[1]);
+ e.countries = line.substring(line.indexOf(",",
+ line.indexOf(",") + 1));
+ this.combinedDatabase.put(e.fromIP, e);
+ }
+ }
+ br.close();
+ this.logger.fine("Finished reading in "
+ + this.combinedDatabaseFile.getAbsolutePath() + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed to read in "
+ + this.combinedDatabaseFile.getAbsolutePath() + "!", e);
+ }
+ }
+ }
+
+ /**
+ * Downloads today's commercial Maxmind GeoIP database, if such a
+ * database exists, and writes it to disk. This method should be called
+ * before importing GeoIP databases from disk if the new database should
+ * be included in the combined database.
+ */
+ public void downloadGeoIPDatabase(String licenseKey) {
+ if (licenseKey == null || licenseKey.length() < 1) {
+ logger.warning("Missing or invalid license key for downloading "
+ + "GeoIP database!");
+ return;
+ }
+
+ /* Find out when we tried downloading the last time to avoid making
+ * too many download attempts. */
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long now = System.currentTimeMillis();
+ if (this.lastDownloadedTime != null) {
+ long lastDownloaded = -1;
+ try {
+ lastDownloaded = dateTimeFormat.parse(this.lastDownloadedTime).
+ getTime();
+ } catch (ParseException e) {
+ logger.log(Level.WARNING, "Could not parse last downloaded "
+ + "time '" + this.lastDownloadedTime + "'. Ignoring.");
+ }
+ if (lastDownloaded + 8L * 60L * 60L * 1000L > now) {
+ logger.finer("Last GeoIP database download not more than 8 "
+ + "hours in the past. Not downloading.");
+ return;
+ }
+ }
+
+ /* Download GeoIP database. */
+ try {
+ logger.fine("Downloading GeoIP database...");
+ this.lastDownloadedTime = dateTimeFormat.format(now);
+ SimpleDateFormat urlDateFormat = new SimpleDateFormat("yyyyMMdd");
+ urlDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String date = urlDateFormat.format(now);
+ String url = "http://www.maxmind.com/app/download_new";
+ + "?edition_id=108&date=" + date + "&suffix=zip&license_key="
+ + licenseKey;
+ URL u = new URL(url);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ int response = huc.getResponseCode();
+ if (response != 200) {
+ logger.fine("Could not download GeoIP database. Response code "
+ + response);
+ return;
+ }
+ BufferedInputStream bis = new BufferedInputStream(
+ huc.getInputStream());
+ ZipInputStream zis = new ZipInputStream(bis);
+ BufferedInputStream bis2 = new BufferedInputStream(zis);
+ ZipEntry entry = null;
+ while ((entry = zis.getNextEntry()) != null) {
+ if (!entry.isDirectory() &&
+ entry.getName().endsWith("GeoIP-108.csv")) {
+ String filename = "geoipdb/GeoIP-108_" + date + ".csv";
+ OutputStream out = new BufferedOutputStream(
+ new FileOutputStream(filename));
+ byte[] buffer = new byte[1024];
+ int len;
+ while ((len = bis2.read(buffer)) >= 0) {
+ out.write(buffer, 0, len);
+ }
+ out.close();
+ }
+ }
+ zis.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not download GeoIP database. "
+ + "Exiting.", e);
+ return;
+ }
+ }
+
+ /**
+ * Imports the GeoIP databases from <code>directory</code> to include
+ * them in the combined GeoIP database.
+ */
+ public void importGeoIPDatabaseFromDisk(String directory) {
+ File databaseDirectory = new File(directory);
+ if (!databaseDirectory.exists()) {
+ return;
+ }
+ for (File databaseFile : databaseDirectory.listFiles()) {
+ String filename = databaseFile.getName();
+ if (!filename.startsWith("GeoIP-108_") ||
+ filename.length() != "GeoIP-108_xxxxxxxx.csv".length() ||
+ !filename.endsWith(".csv")) {
+ continue;
+ }
+ String date = filename.substring("GeoIP-108_".length(),
+ "GeoIP-108_xxxxxxxx".length());
+ if (allDatabases.contains(date)) {
+ continue;
+ }
+ this.combinedDatabaseModified = true;
+ this.logger.fine("Reading in " + filename);
+ String emptyCountryString = "";
+ for (int i = 0; i < this.allDatabases.size(); i++) {
+ emptyCountryString += ",ZZ";
+ }
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ databaseFile));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("Copyright") ||
+ line.startsWith("\"begin")) {
+ continue;
+ }
+ String lineWithoutQuotes = line.replaceAll("\"", "");
+ String[] parts = lineWithoutQuotes.split(",");
+ lineWithoutQuotes = null; // does this help GC?
+ long fromIP = Long.parseLong(parts[2]);
+ long toIP = Long.parseLong(parts[3]);
+ String countryCode = parts[4];
+ SortedMap<Long, DatabaseEntry> submap =
+ combinedDatabase.headMap(toIP + 1L);
+ if (!submap.headMap(fromIP + 1L).isEmpty()) {
+ submap = submap.tailMap(submap.headMap(fromIP + 1L).
+ lastKey());
+ }
+ Set<DatabaseEntry> newEntries = new HashSet<DatabaseEntry>();
+ for (DatabaseEntry e : submap.values()) {
+ while (fromIP <= toIP && fromIP <= e.toIP &&
+ toIP >= e.fromIP) {
+ if (fromIP < e.fromIP) {
+ // duplicate entry: new entry fromIP-e.fromIP, set fromIP
+ // to e.fromIP
+ DatabaseEntry e1 = new DatabaseEntry();
+ e1.fromIP = fromIP;
+ e1.toIP = e.fromIP - 1L;
+ e1.countries = e.countries;
+ newEntries.add(e1);
+ fromIP = e.fromIP;
+ } else if (fromIP > e.fromIP) {
+ // split off existing entry; don't add yet
+ DatabaseEntry e1 = new DatabaseEntry();
+ e1.fromIP = e.fromIP;
+ e1.toIP = fromIP - 1L;
+ e1.countries = e.countries;
+ newEntries.add(e1);
+ e.fromIP = fromIP;
+ newEntries.add(e);
+ } else if (toIP < e.toIP) {
+ // split and add to first half
+ DatabaseEntry e1 = new DatabaseEntry();
+ e1.fromIP = toIP + 1L;
+ e1.toIP = e.toIP;
+ e1.countries = e.countries;
+ newEntries.add(e1);
+ e.toIP = e1.fromIP - 1L;
+ e.countries += "," + countryCode;
+ fromIP = toIP + 1L;
+ } else if (toIP >= e.toIP) {
+ // add to this entry and done, right?
+ e.countries += "," + countryCode;
+ fromIP = toIP + 1L;
+ }
+ }
+ }
+ if (fromIP <= toIP) {
+ DatabaseEntry entry = new DatabaseEntry();
+ entry.fromIP = fromIP;
+ entry.toIP = toIP;
+ entry.countries = emptyCountryString + "," + countryCode;
+ newEntries.add(entry);
+ fromIP = toIP + 1L;
+ }
+ for (DatabaseEntry e : newEntries) {
+ this.combinedDatabase.put(e.fromIP, e);
+ }
+ }
+ this.allDatabases.add(date);
+ for (DatabaseEntry e : this.combinedDatabase.values()) {
+ if (e.countries.substring(1).split(",").length <
+ this.allDatabases.size()) {
+ e.countries += ",ZZ";
+ }
+ }
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not import GeoIP database "
+ + "from file " + databaseFile.getAbsolutePath()
+ + ". This might leave us with an inconsistent state!");
+ }
+ }
+ }
+
+ public void writeCombinedDatabase() {
+ if (!combinedDatabaseModified) {
+ return;
+ }
+ try {
+ this.logger.fine("Writing "
+ + this.combinedDatabaseFile.getAbsolutePath() + "...");
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.combinedDatabaseFile));
+ bw.write("lastDownload " + this.lastDownloadedTime + "\n");
+ bw.write("beginIpNum,endIpNum");
+ for (String d : allDatabases) {
+ bw.write("," + d);
+ }
+ bw.write("\n");
+ for (DatabaseEntry e : this.combinedDatabase.values()) {
+ bw.write(e.fromIP + "," + e.toIP + e.countries + "\n");
+ }
+ bw.close();
+ this.logger.fine("Finished writing "
+ + this.combinedDatabaseFile.getAbsolutePath() + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed to write "
+ + this.combinedDatabaseFile.getAbsolutePath() + "!");
+ }
+ }
+
+ /**
+ * Returns the uppercase two-letter country code that was assigned to
+ * <code>ipAddress</code> (in dotted notation) in the most recent
+ * commercial Maxmind GeoIP database published at least 1 day before
+ * <code>date</code> (in the format yyyy-MM-dd).
+ */
+ public String getCountryForIP(String ipAddress, String date) {
+ String[] parts = ipAddress.split("\\.");
+ long ipNum = Long.parseLong(parts[0]) * 256 * 256 * 256 +
+ Long.parseLong(parts[1]) * 256 * 256 +
+ Long.parseLong(parts[2]) * 256 + Long.parseLong(parts[3]);
+ String countries = null;
+ if (this.combinedDatabase.containsKey(ipNum)) {
+ countries = this.combinedDatabase.get(ipNum).countries;
+ } else if (!this.combinedDatabase.headMap(ipNum).isEmpty()) {
+ countries = this.combinedDatabase.get(this.combinedDatabase.headMap(
+ ipNum).lastKey()).countries;
+ } else {
+ return "ZZ";
+ }
+ String dateShort = date.substring(0, 4) + date.substring(5, 7)
+ + date.substring(8, 10);
+ SortedSet<String> subset = new TreeSet<String>(this.allDatabases).
+ headSet(dateShort);
+ if (subset.isEmpty()) {
+ return "ZZ";
+ }
+ int index = allDatabases.indexOf(subset.last());
+ return countries.substring(1).split(",")[index];
+ }
+}
diff --git a/src/Main.java b/src/Main.java
index 0990ca1..08112f6 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -95,6 +95,16 @@ public class Main {
aw = null;
}
+ // Import/download GeoIP databases
+ GeoIPDatabaseManager gd = new GeoIPDatabaseManager();
+ if (config.getDownloadGeoIPDatabase()) {
+ gd.downloadGeoIPDatabase(config.getMaxmindLicenseKey());
+ }
+ if (config.getImportGeoIPDatabases()) {
+ gd.importGeoIPDatabaseFromDisk("geoipdb/");
+ gd.writeCombinedDatabase();
+ }
+
// Prepare bridge descriptor parser
BridgeDescriptorParser bdp = config.getWriteConsensusStats() &&
config.getWriteBridgeStats() ? new BridgeDescriptorParser(
--
1.6.5