[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [metrics-tasks/master] Generalize database to AS numbers (#6471).
commit 5ffcbdc3713b4e3e698b7fee147e5301d242ef80
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Sat Nov 3 21:39:13 2012 -0400
Generalize database to AS numbers (#6471).
---
.../src/org/torproject/task6471/Countries.java | 284 --------------------
.../java/src/org/torproject/task6471/Database.java | 49 +---
.../src/org/torproject/task6471/DatabaseImpl.java | 57 +---
.../torproject/task6471/DatabaseImporterImpl.java | 59 ++---
.../task6471/DatabasePerformanceExample.java | 17 +-
.../src/org/torproject/task6471/DatabaseTest.java | 103 ++++----
6 files changed, 114 insertions(+), 455 deletions(-)
diff --git a/task-6471/java/src/org/torproject/task6471/Countries.java b/task-6471/java/src/org/torproject/task6471/Countries.java
deleted file mode 100644
index a70c4bf..0000000
--- a/task-6471/java/src/org/torproject/task6471/Countries.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.task6471;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class Countries {
-
- private static Countries instance = new Countries();
-
- public static Countries getInstance() {
- return Countries.instance;
- }
-
- /* List of arrays of length 2, containing country codes at [0] and
- * country names at [1], alphabetically ordered by country names. */
- private List<String[]> knownCountries;
-
- private Countries() {
- this.knownCountries = new ArrayList<String[]>();
- this.knownCountries.add("af;Afghanistan".split(";"));
- this.knownCountries.add("ax;Aland Islands".split(";"));
- this.knownCountries.add("al;Albania".split(";"));
- this.knownCountries.add("dz;Algeria".split(";"));
- this.knownCountries.add("as;American Samoa".split(";"));
- this.knownCountries.add("ad;Andorra".split(";"));
- this.knownCountries.add("ao;Angola".split(";"));
- this.knownCountries.add("ai;Anguilla".split(";"));
- this.knownCountries.add("aq;Antarctica".split(";"));
- this.knownCountries.add("ag;Antigua and Barbuda".split(";"));
- this.knownCountries.add("ar;Argentina".split(";"));
- this.knownCountries.add("am;Armenia".split(";"));
- this.knownCountries.add("aw;Aruba".split(";"));
- this.knownCountries.add("au;Australia".split(";"));
- this.knownCountries.add("at;Austria".split(";"));
- this.knownCountries.add("az;Azerbaijan".split(";"));
- this.knownCountries.add("bs;Bahamas".split(";"));
- this.knownCountries.add("bh;Bahrain".split(";"));
- this.knownCountries.add("bd;Bangladesh".split(";"));
- this.knownCountries.add("bb;Barbados".split(";"));
- this.knownCountries.add("by;Belarus".split(";"));
- this.knownCountries.add("be;Belgium".split(";"));
- this.knownCountries.add("bz;Belize".split(";"));
- this.knownCountries.add("bj;Benin".split(";"));
- this.knownCountries.add("bm;Bermuda".split(";"));
- this.knownCountries.add("bt;Bhutan".split(";"));
- this.knownCountries.add("bo;Bolivia".split(";"));
- this.knownCountries.add("ba;Bosnia and Herzegovina".split(";"));
- this.knownCountries.add("bw;Botswana".split(";"));
- this.knownCountries.add("bv;Bouvet Island".split(";"));
- this.knownCountries.add("br;Brazil".split(";"));
- this.knownCountries.add("io;British Indian Ocean Territory".
- split(";"));
- this.knownCountries.add("bn;Brunei".split(";"));
- this.knownCountries.add("bg;Bulgaria".split(";"));
- this.knownCountries.add("bf;Burkina Faso".split(";"));
- this.knownCountries.add("mm;Burma".split(";"));
- this.knownCountries.add("bi;Burundi".split(";"));
- this.knownCountries.add("kh;Cambodia".split(";"));
- this.knownCountries.add("cm;Cameroon".split(";"));
- this.knownCountries.add("ca;Canada".split(";"));
- this.knownCountries.add("cv;Cape Verde".split(";"));
- this.knownCountries.add("ky;Cayman Islands".split(";"));
- this.knownCountries.add("cf;Central African Republic".split(";"));
- this.knownCountries.add("td;Chad".split(";"));
- this.knownCountries.add("cl;Chile".split(";"));
- this.knownCountries.add("cn;China".split(";"));
- this.knownCountries.add("cx;Christmas Island".split(";"));
- this.knownCountries.add("cc;Cocos (Keeling) Islands".split(";"));
- this.knownCountries.add("co;Colombia".split(";"));
- this.knownCountries.add("km;Comoros".split(";"));
- this.knownCountries.add("cd;Congo, The Democratic Republic of the".
- split(";"));
- this.knownCountries.add("cg;Congo".split(";"));
- this.knownCountries.add("ck;Cook Islands".split(";"));
- this.knownCountries.add("cr;Costa Rica".split(";"));
- this.knownCountries.add("ci:Côte d'Ivoire".split(":"));
- this.knownCountries.add("hr;Croatia".split(";"));
- this.knownCountries.add("cu;Cuba".split(";"));
- this.knownCountries.add("cy;Cyprus".split(";"));
- this.knownCountries.add("cz;Czech Republic".split(";"));
- this.knownCountries.add("dk;Denmark".split(";"));
- this.knownCountries.add("dj;Djibouti".split(";"));
- this.knownCountries.add("dm;Dominica".split(";"));
- this.knownCountries.add("do;Dominican Republic".split(";"));
- this.knownCountries.add("ec;Ecuador".split(";"));
- this.knownCountries.add("eg;Egypt".split(";"));
- this.knownCountries.add("sv;El Salvador".split(";"));
- this.knownCountries.add("gq;Equatorial Guinea".split(";"));
- this.knownCountries.add("er;Eritrea".split(";"));
- this.knownCountries.add("ee;Estonia".split(";"));
- this.knownCountries.add("et;Ethiopia".split(";"));
- this.knownCountries.add("fk;Falkland Islands (Malvinas)".split(";"));
- this.knownCountries.add("fo;Faroe Islands".split(";"));
- this.knownCountries.add("fj;Fiji".split(";"));
- this.knownCountries.add("fi;Finland".split(";"));
- this.knownCountries.add("fx;France, Metropolitan".split(";"));
- this.knownCountries.add("fr;France".split(";"));
- this.knownCountries.add("gf;French Guiana".split(";"));
- this.knownCountries.add("pf;French Polynesia".split(";"));
- this.knownCountries.add("tf;French Southern Territories".split(";"));
- this.knownCountries.add("ga;Gabon".split(";"));
- this.knownCountries.add("gm;Gambia".split(";"));
- this.knownCountries.add("ge;Georgia".split(";"));
- this.knownCountries.add("de;Germany".split(";"));
- this.knownCountries.add("gh;Ghana".split(";"));
- this.knownCountries.add("gi;Gibraltar".split(";"));
- this.knownCountries.add("gr;Greece".split(";"));
- this.knownCountries.add("gl;Greenland".split(";"));
- this.knownCountries.add("gd;Grenada".split(";"));
- this.knownCountries.add("gp;Guadeloupe".split(";"));
- this.knownCountries.add("gu;Guam".split(";"));
- this.knownCountries.add("gt;Guatemala".split(";"));
- this.knownCountries.add("gg;Guernsey".split(";"));
- this.knownCountries.add("gn;Guinea".split(";"));
- this.knownCountries.add("gw;Guinea-Bissau".split(";"));
- this.knownCountries.add("gy;Guyana".split(";"));
- this.knownCountries.add("ht;Haiti".split(";"));
- this.knownCountries.add("hm;Heard Island and McDonald Islands".
- split(";"));
- this.knownCountries.add("va;Vatican City".split(";"));
- this.knownCountries.add("hn;Honduras".split(";"));
- this.knownCountries.add("hk;Hong Kong".split(";"));
- this.knownCountries.add("hu;Hungary".split(";"));
- this.knownCountries.add("is;Iceland".split(";"));
- this.knownCountries.add("in;India".split(";"));
- this.knownCountries.add("id;Indonesia".split(";"));
- this.knownCountries.add("ir;Iran".split(";"));
- this.knownCountries.add("iq;Iraq".split(";"));
- this.knownCountries.add("ie;Ireland".split(";"));
- this.knownCountries.add("im;Isle of Man".split(";"));
- this.knownCountries.add("il;Israel".split(";"));
- this.knownCountries.add("it;Italy".split(";"));
- this.knownCountries.add("jm;Jamaica".split(";"));
- this.knownCountries.add("jp;Japan".split(";"));
- this.knownCountries.add("je;Jersey".split(";"));
- this.knownCountries.add("jo;Jordan".split(";"));
- this.knownCountries.add("kz;Kazakhstan".split(";"));
- this.knownCountries.add("ke;Kenya".split(";"));
- this.knownCountries.add("ki;Kiribati".split(";"));
- this.knownCountries.add("kp;North Korea".split(";"));
- this.knownCountries.add("kr;Korea, Republic of".split(";"));
- this.knownCountries.add("kw;Kuwait".split(";"));
- this.knownCountries.add("kg;Kyrgyzstan".split(";"));
- this.knownCountries.add("la;Laos".split(";"));
- this.knownCountries.add("lv;Latvia".split(";"));
- this.knownCountries.add("lb;Lebanon".split(";"));
- this.knownCountries.add("ls;Lesotho".split(";"));
- this.knownCountries.add("lr;Liberia".split(";"));
- this.knownCountries.add("ly;Libya".split(";"));
- this.knownCountries.add("li;Liechtenstein".split(";"));
- this.knownCountries.add("lt;Lithuania".split(";"));
- this.knownCountries.add("lu;Luxembourg".split(";"));
- this.knownCountries.add("mo;Macau".split(";"));
- this.knownCountries.add("mk;Macedonia".split(";"));
- this.knownCountries.add("mg;Madagascar".split(";"));
- this.knownCountries.add("mw;Malawi".split(";"));
- this.knownCountries.add("my;Malaysia".split(";"));
- this.knownCountries.add("mv;Maldives".split(";"));
- this.knownCountries.add("ml;Mali".split(";"));
- this.knownCountries.add("mt;Malta".split(";"));
- this.knownCountries.add("mh;Marshall Islands".split(";"));
- this.knownCountries.add("mq;Martinique".split(";"));
- this.knownCountries.add("mr;Mauritania".split(";"));
- this.knownCountries.add("mu;Mauritius".split(";"));
- this.knownCountries.add("yt;Mayotte".split(";"));
- this.knownCountries.add("mx;Mexico".split(";"));
- this.knownCountries.add("fm;Micronesia, Federated States of".
- split(";"));
- this.knownCountries.add("md;Moldova, Republic of".split(";"));
- this.knownCountries.add("mc;Monaco".split(";"));
- this.knownCountries.add("mn;Mongolia".split(";"));
- this.knownCountries.add("me;Montenegro".split(";"));
- this.knownCountries.add("ms;Montserrat".split(";"));
- this.knownCountries.add("ma;Morocco".split(";"));
- this.knownCountries.add("mz;Mozambique".split(";"));
- this.knownCountries.add("mm;Burma".split(";"));
- this.knownCountries.add("na;Namibia".split(";"));
- this.knownCountries.add("nr;Nauru".split(";"));
- this.knownCountries.add("np;Nepal".split(";"));
- this.knownCountries.add("an;Netherlands Antilles".split(";"));
- this.knownCountries.add("nl;Netherlands".split(";"));
- this.knownCountries.add("nc;New Caledonia".split(";"));
- this.knownCountries.add("nz;New Zealand".split(";"));
- this.knownCountries.add("ni;Nicaragua".split(";"));
- this.knownCountries.add("ne;Niger".split(";"));
- this.knownCountries.add("ng;Nigeria".split(";"));
- this.knownCountries.add("nu;Niue".split(";"));
- this.knownCountries.add("nf;Norfolk Island".split(";"));
- this.knownCountries.add("mp;Northern Mariana Islands".split(";"));
- this.knownCountries.add("no;Norway".split(";"));
- this.knownCountries.add("om;Oman".split(";"));
- this.knownCountries.add("pk;Pakistan".split(";"));
- this.knownCountries.add("pw;Palau".split(";"));
- this.knownCountries.add("ps;Palestinian Territory".split(";"));
- this.knownCountries.add("pa;Panama".split(";"));
- this.knownCountries.add("pg;Papua New Guinea".split(";"));
- this.knownCountries.add("py;Paraguay".split(";"));
- this.knownCountries.add("pe;Peru".split(";"));
- this.knownCountries.add("ph;Philippines".split(";"));
- this.knownCountries.add("pn;Pitcairn Islands".split(";"));
- this.knownCountries.add("pl;Poland".split(";"));
- this.knownCountries.add("pt;Portugal".split(";"));
- this.knownCountries.add("pr;Puerto Rico".split(";"));
- this.knownCountries.add("qa;Qatar".split(";"));
- this.knownCountries.add("re;Reunion".split(";"));
- this.knownCountries.add("ro;Romania".split(";"));
- this.knownCountries.add("ru;Russia".split(";"));
- this.knownCountries.add("rw;Rwanda".split(";"));
- this.knownCountries.add("bl;Saint Bartelemey".split(";"));
- this.knownCountries.add("sh;Saint Helena".split(";"));
- this.knownCountries.add("kn;Saint Kitts and Nevis".split(";"));
- this.knownCountries.add("lc;Saint Lucia".split(";"));
- this.knownCountries.add("mf;Saint Martin".split(";"));
- this.knownCountries.add("pm;Saint Pierre and Miquelon".split(";"));
- this.knownCountries.add("vc;Saint Vincent and the Grenadines".
- split(";"));
- this.knownCountries.add("ws;Samoa".split(";"));
- this.knownCountries.add("sm;San Marino".split(";"));
- this.knownCountries.add("st:São Tomé and Príncipe".
- split(":"));
- this.knownCountries.add("sa;Saudi Arabia".split(";"));
- this.knownCountries.add("sn;Senegal".split(";"));
- this.knownCountries.add("rs;Serbia".split(";"));
- this.knownCountries.add("sc;Seychelles".split(";"));
- this.knownCountries.add("sl;Sierra Leone".split(";"));
- this.knownCountries.add("sg;Singapore".split(";"));
- this.knownCountries.add("sk;Slovakia".split(";"));
- this.knownCountries.add("si;Slovenia".split(";"));
- this.knownCountries.add("sb;Solomon Islands".split(";"));
- this.knownCountries.add("so;Somalia".split(";"));
- this.knownCountries.add("za;South Africa".split(";"));
- this.knownCountries.add(("gs;South Georgia and the South Sandwich "
- + "Islands").split(";"));
- this.knownCountries.add("es;Spain".split(";"));
- this.knownCountries.add("lk;Sri Lanka".split(";"));
- this.knownCountries.add("sd;Sudan".split(";"));
- this.knownCountries.add("sr;Suriname".split(";"));
- this.knownCountries.add("sj;Svalbard and Jan Mayen".split(";"));
- this.knownCountries.add("sz;Swaziland".split(";"));
- this.knownCountries.add("se;Sweden".split(";"));
- this.knownCountries.add("ch;Switzerland".split(";"));
- this.knownCountries.add("sy;Syrian Arab Republic".split(";"));
- this.knownCountries.add("tw;Taiwan".split(";"));
- this.knownCountries.add("tj;Tajikistan".split(";"));
- this.knownCountries.add("tz;Tanzania, United Republic of".split(";"));
- this.knownCountries.add("th;Thailand".split(";"));
- this.knownCountries.add("tl;East Timor".split(";"));
- this.knownCountries.add("tg;Togo".split(";"));
- this.knownCountries.add("tk;Tokelau".split(";"));
- this.knownCountries.add("to;Tonga".split(";"));
- this.knownCountries.add("tt;Trinidad and Tobago".split(";"));
- this.knownCountries.add("tn;Tunisia".split(";"));
- this.knownCountries.add("tr;Turkey".split(";"));
- this.knownCountries.add("tm;Turkmenistan".split(";"));
- this.knownCountries.add("tc;Turks and Caicos Islands".split(";"));
- this.knownCountries.add("tv;Tuvalu".split(";"));
- this.knownCountries.add("ug;Uganda".split(";"));
- this.knownCountries.add("ua;Ukraine".split(";"));
- this.knownCountries.add("ae;United Arab Emirates".split(";"));
- this.knownCountries.add("gb;United Kingdom".split(";"));
- this.knownCountries.add("um;United States Minor Outlying Islands".
- split(";"));
- this.knownCountries.add("us;United States".split(";"));
- this.knownCountries.add("uy;Uruguay".split(";"));
- this.knownCountries.add("uz;Uzbekistan".split(";"));
- this.knownCountries.add("vu;Vanuatu".split(";"));
- this.knownCountries.add("ve;Venezuela".split(";"));
- this.knownCountries.add("vn;Vietnam".split(";"));
- this.knownCountries.add("vg;Virgin Islands, British".split(";"));
- this.knownCountries.add("vi;Virgin Islands, U.S.".split(";"));
- this.knownCountries.add("wf;Wallis and Futuna".split(";"));
- this.knownCountries.add("eh;Western Sahara".split(";"));
- this.knownCountries.add("ye;Yemen".split(";"));
- this.knownCountries.add("zm;Zambia".split(";"));
- this.knownCountries.add("zw;Zimbabwe".split(";"));
- }
-
- public List<String[]> getCountryList() {
- return this.knownCountries;
- }
-}
-
diff --git a/task-6471/java/src/org/torproject/task6471/Database.java b/task-6471/java/src/org/torproject/task6471/Database.java
index 62a5e15..448cda6 100644
--- a/task-6471/java/src/org/torproject/task6471/Database.java
+++ b/task-6471/java/src/org/torproject/task6471/Database.java
@@ -2,15 +2,16 @@
package org.torproject.task6471;
/**
- * Database storing multiple GeoIP databases and supporting efficient
- * ip-to-country lookups in the most recent of those databases for any
- * given date.
+ * Database storing multiple GeoIP or ASN databases and supporting
+ * efficient ip-to-country-code or ip-to-AS-number lookups in the most
+ * recent of those databases for any given date.
*
- * A typical query for this GeoIP database is: "to which country was IPv4
- * address 1.2.3.4 assigned on date 20120912?" This query is answered by
- * looking at the entries from the most recent database published on or
- * before 20120912. If the earliest known database was published after
- * 20120912, the earliest known database is used to resolve the request.
+ * A typical query for this GeoIP database is: "to which country code was
+ * IPv4 address 1.2.3.4 assigned on date 20120912?" This query is
+ * answered by looking at the entries from the most recent database
+ * published on or before 20120912. If the earliest known database was
+ * published after 20120912, the earliest known database is used to
+ * resolve the request.
*/
public interface Database {
@@ -24,35 +25,13 @@ public interface Database {
public boolean loadCombinedDatabases(String path);
/**
- * Query the database for the country code assigned to an IPv4 address
- * and date.
+ * Query the database for the country code or AS number assigned to an
+ * IPv4 address on a given date.
*
* @param address IPv4 address in dotted-quad notation.
* @param date Assignment date in format yyyymmdd.
- * @return Assigned country code, or null if no assignment could be
- * found.
+ * @return Assigned country code or AS number, or null if no assignment
+ * could be found.
*/
- public String lookupCountryCodeFromIpv4AddressAndDate(String address,
- String date);
-
- /**
- * Resolve the country code to the corresponding country name.
- *
- * @param countryCode Country code to be resolved.
- * @return Country name.
- */
- public String getCountryNameForCountryCode(String countryCode);
-
- /**
- * Query the database for the country name assigned to an IPv4 address
- * and date. This is a convenience method that first looks up the
- * country code and then resolves it to the country name.
- *
- * @param address IPv4 address in dotted-quad notation.
- * @param date Assignment date in format yyyymmdd.
- * @return Assigned country name, or null if no assignment could be
- * found.
- */
- public String lookupCountryNameFromIpv4AddressAndDate(String address,
- String date);
+ public String lookupIpv4AddressAndDate(String address, String date);
}
diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java
index 7fa6f13..364f0c7 100644
--- a/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java
+++ b/task-6471/java/src/org/torproject/task6471/DatabaseImpl.java
@@ -8,7 +8,6 @@ import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Collections;
-import java.util.List;
import java.util.Map;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -17,7 +16,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
/**
- * Implementation of database holding multiple GeoIP databases with
+ * Implementation of database holding multiple GeoIP or ASN databases with
* special focus on lookup performance, import performance, and memory
* consumption (in that order).
*
@@ -26,7 +25,7 @@ import java.util.TreeSet;
* the start IPv4 address in the higher bits and the first database
* publication date containing that range in the lower bits. The tree
* element itself contains the end IPv4 address, last database publication
- * date, and country code.
+ * date, and country code or AS number.
*
* Lookups for a given address and random date only require iterating
* backwards over ranges with start address smaller than or equaling the
@@ -42,20 +41,19 @@ public class DatabaseImpl implements Database {
/**
* Tree element containing an end IPv4 address, last database date,
- * last database index, and country code. Start IPv4 address and first
- * database date are encoded in the key under which the element is
- * stored.
+ * last database index, and country code or AS number. Start IPv4
+ * address and first database date are encoded in the key under which
+ * the element is stored.
*/
protected static class TreeElement {
protected long endAddress;
protected int lastDbDate;
- protected String countryCode;
+ protected String code;
protected boolean modifiedInLastImport;
- protected TreeElement(long endAddress, int lastDbDate,
- String countryCode) {
+ protected TreeElement(long endAddress, int lastDbDate, String code) {
this.endAddress = endAddress;
this.lastDbDate = lastDbDate;
- this.countryCode = countryCode;
+ this.code = code;
this.modifiedInLastImport = true;
}
}
@@ -86,7 +84,7 @@ public class DatabaseImpl implements Database {
* Look up address and date by iterating backwards over possibly
* matching ranges.
*/
- public String lookupCountryCodeFromIpv4AddressAndDate(
+ public String lookupIpv4AddressAndDate(
String addressString, String dateString) {
this.addressLookups++;
@@ -124,8 +122,8 @@ public class DatabaseImpl implements Database {
}
/* Both address and date ranges match, so return the assigned
- * country code. */
- return e.getValue().countryCode;
+ * code. */
+ return e.getValue().code;
}
/* No ranges (left) to look at. We don't have what we were looking
@@ -222,7 +220,7 @@ public class DatabaseImpl implements Database {
sb.append(String.format("%n %s %s %s %s %s",
convertKeyToAddressString(e.getKey()),
convertAddressNumberToString(e.getValue().endAddress),
- e.getValue().countryCode,
+ e.getValue().code,
convertKeyToDateString(e.getKey()),
convertDateNumberToString(e.getValue().lastDbDate)));
if (--entries <= 0) {
@@ -259,12 +257,12 @@ public class DatabaseImpl implements Database {
String[] parts = line.split(",");
long startAddress = convertAddressStringToNumber(parts[0]);
long endAddress = convertAddressStringToNumber(parts[1]);
- String countryCode = parts[2];
+ String code = parts[2];
int firstDbDate = convertDateStringToNumber(parts[3]);
int lastDbDate = convertDateStringToNumber(parts[4]);
this.ranges.put(convertAddressAndDateToKey(startAddress,
firstDbDate), new TreeElement(endAddress, lastDbDate,
- countryCode));
+ code));
}
}
br.close();
@@ -273,31 +271,4 @@ public class DatabaseImpl implements Database {
}
return true;
}
-
- /** Mapping from country codes to country names. */
- private static SortedMap<String, String> countryNames;
- static {
- countryNames = new TreeMap<String, String>();
- List<String[]> countryList = Countries.getInstance().getCountryList();
- for (String[] country : countryList) {
- countryNames.put(country[0], country[1]);
- }
- }
-
- /** Resolve country codes to country names. */
- public String getCountryNameForCountryCode(String countryCode) {
- if (countryCode == null) {
- return null;
- } else {
- return countryNames.get(countryCode);
- }
- }
-
- /** Lookup IPv4 address and date and return the country name. */
- public String lookupCountryNameFromIpv4AddressAndDate(
- String addressString, String dateString) {
- return getCountryNameForCountryCode(
- lookupCountryCodeFromIpv4AddressAndDate(addressString,
- dateString));
- }
}
diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java
index 6937cfa..4795883 100644
--- a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java
+++ b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java
@@ -94,10 +94,14 @@ public class DatabaseImporterImpl extends DatabaseImpl
/* TODO Add support for IPv6 addresses. */
continue;
}
- String countryCode = parts[1].toLowerCase();
+ String code = parts[1].toLowerCase();
+ if (code.length() != 2) {
+ /* Don't import illegal range. */
+ continue;
+ }
String startAddressString = parts[3];
long addresses = Long.parseLong(parts[4]);
- this.addRange(databaseFileName, countryCode, startAddressString,
+ this.addRange(databaseFileName, code, startAddressString,
addresses);
}
br.close();
@@ -123,14 +127,9 @@ public class DatabaseImporterImpl extends DatabaseImpl
* is called prior to any lookupAddress() calls. No further checks are
* performed that the tree is repaired before looking up an address.
*/
- void addRange(String databaseFileName, String countryCode,
+ void addRange(String databaseFileName, String code,
String startAddressString, long addresses) {
- if (countryCode.length() != 2) {
- /* Don't import illegal range. */
- return;
- }
-
this.rangeImports++;
String databaseDateString =
databaseFileName.substring(databaseFileName.length() - 8);
@@ -151,8 +150,8 @@ public class DatabaseImporterImpl extends DatabaseImpl
* We shouldn't mess with the tree directly while iterating over it,
* so let's for now only calculate what changes we want to make. */
SortedMap<Long, TreeElement> updateElements =
- this.getUpdatesForAddingRange(databaseDate, countryCode,
- startAddress, endAddress);
+ this.getUpdatesForAddingRange(databaseDate, code, startAddress,
+ endAddress);
/* Apply updates. Elements with non-null values are added, elements
* with null values are removed. */
@@ -169,8 +168,7 @@ public class DatabaseImporterImpl extends DatabaseImpl
* Calculate necessary changes to the tree to add a range.
*/
private SortedMap<Long, TreeElement> getUpdatesForAddingRange(
- int databaseDate, String countryCode, long startAddress,
- long endAddress) {
+ int databaseDate, String code, long startAddress, long endAddress) {
/* Keep updates in a single tree where non-null values will later be
* added, possibly replacing existing elements, and null values will
@@ -205,7 +203,7 @@ public class DatabaseImporterImpl extends DatabaseImpl
long eEndAddress = e.getValue().endAddress;
int eFirstDbDate = convertKeyToDate(e.getKey());
int eLastDbDate = e.getValue().lastDbDate;
- String eCountryCode = e.getValue().countryCode;
+ String eCode = e.getValue().code;
/* If the next (partial) range starts after the current element
* ends, add the new range. */
@@ -213,7 +211,7 @@ public class DatabaseImporterImpl extends DatabaseImpl
nextEndAddress >= startAddress) {
updateElements.put(convertAddressAndDateToKey(nextStartAddress,
nextFirstDbDate), new TreeElement(nextEndAddress,
- nextLastDbDate, countryCode));
+ nextLastDbDate, code));
nextEndAddress = nextStartAddress - 1L;
nextStartAddress = startAddress;
nextFirstDbDate = databaseDate;
@@ -226,7 +224,7 @@ public class DatabaseImporterImpl extends DatabaseImpl
nextEndAddress >= startAddress) {
updateElements.put(convertAddressAndDateToKey(eEndAddress + 1L,
databaseDate), new TreeElement(nextEndAddress, databaseDate,
- countryCode));
+ code));
nextEndAddress = eEndAddress;
nextStartAddress = startAddress;
nextFirstDbDate = databaseDate;
@@ -247,10 +245,10 @@ public class DatabaseImporterImpl extends DatabaseImpl
if (eStartAddress <= endAddress && eEndAddress > endAddress) {
updateElements.put(convertAddressAndDateToKey(endAddress + 1L,
eFirstDbDate), new TreeElement(eEndAddress, eLastDbDate,
- eCountryCode));
+ eCode));
updateElements.put(convertAddressAndDateToKey(eStartAddress,
eFirstDbDate), new TreeElement(endAddress, eLastDbDate,
- eCountryCode));
+ eCode));
eEndAddress = endAddress;
}
@@ -260,10 +258,10 @@ public class DatabaseImporterImpl extends DatabaseImpl
if (eStartAddress < startAddress && eEndAddress >= startAddress) {
updateElements.put(convertAddressAndDateToKey(eStartAddress,
eFirstDbDate), new TreeElement(startAddress - 1L, eLastDbDate,
- eCountryCode));
+ eCode));
updateElements.put(convertAddressAndDateToKey(startAddress,
eFirstDbDate), new TreeElement(eEndAddress, eLastDbDate,
- eCountryCode));
+ eCode));
eStartAddress = startAddress;
}
@@ -272,11 +270,11 @@ public class DatabaseImporterImpl extends DatabaseImpl
nextStartAddress = eStartAddress;
nextEndAddress = eEndAddress;
- /* If the range is already contained and has the same country code,
- * mark it as updated. If it's contained with a different country
- * code, ignore the update. */
+ /* If the range is already contained and has the same code, mark it
+ * as updated. If it's contained with a different code, ignore the
+ * update. */
if (eFirstDbDate <= databaseDate && eLastDbDate >= databaseDate) {
- if (eCountryCode.equals(countryCode)) {
+ if (eCode.equals(code)) {
nextFirstDbDate = eFirstDbDate;
nextLastDbDate = eLastDbDate;
} else {
@@ -288,7 +286,7 @@ public class DatabaseImporterImpl extends DatabaseImpl
/* See if we can merge the new range with the previous or next
* range. If so, extend our database range and mark the existing
* element for deletion. */
- if (eCountryCode.equals(countryCode)) {
+ if (eCode.equals(code)) {
if (eLastDbDate == previousDatabaseDate) {
nextFirstDbDate = eFirstDbDate;
updateElements.put(convertAddressAndDateToKey(eStartAddress,
@@ -306,7 +304,7 @@ public class DatabaseImporterImpl extends DatabaseImpl
while (nextEndAddress >= startAddress) {
updateElements.put(convertAddressAndDateToKey(nextStartAddress,
nextFirstDbDate), new TreeElement(nextEndAddress,
- nextLastDbDate, countryCode));
+ nextLastDbDate, code));
nextEndAddress = nextStartAddress - 1L;
nextStartAddress = startAddress;
nextFirstDbDate = databaseDate;
@@ -355,7 +353,7 @@ public class DatabaseImporterImpl extends DatabaseImpl
int eLastDbDate = e.getValue().lastDbDate;
long eStartAddress = convertKeyToAddress(e.getKey());
long eEndAddress = e.getValue().endAddress;
- String eCountryCode = e.getValue().countryCode;
+ String eCode = e.getValue().code;
int start = eFirstDbDate, end = eFirstDbDate;
for (int cur : this.databaseDates.tailSet(eFirstDbDate)) {
if (cur > eLastDbDate) {
@@ -364,8 +362,7 @@ public class DatabaseImporterImpl extends DatabaseImpl
if (cur == addedDatabaseDate) {
if (start >= 0 && end >= 0) {
updateElements.put(convertAddressAndDateToKey(eStartAddress,
- start), new TreeElement(eEndAddress, end,
- eCountryCode));
+ start), new TreeElement(eEndAddress, end, eCode));
start = end = -1;
}
} else if (start < 0) {
@@ -376,7 +373,7 @@ public class DatabaseImporterImpl extends DatabaseImpl
}
if (start >= 0 && end >= 0) {
updateElements.put(convertAddressAndDateToKey(eStartAddress,
- start), new TreeElement(eEndAddress, end, eCountryCode));
+ start), new TreeElement(eEndAddress, end, eCode));
}
}
}
@@ -422,7 +419,7 @@ public class DatabaseImporterImpl extends DatabaseImpl
bw.write(String.format("%s,%s,%s,%s,%s%n",
convertKeyToAddressString(e.getKey()),
convertAddressNumberToString(e.getValue().endAddress),
- e.getValue().countryCode,
+ e.getValue().code,
convertKeyToDateString(e.getKey()),
convertDateNumberToString(e.getValue().lastDbDate)));
}
@@ -452,7 +449,7 @@ public class DatabaseImporterImpl extends DatabaseImpl
sb.append(String.format("%n %s %s %s %s %s",
convertKeyToAddressString(e.getKey()),
convertAddressNumberToString(e.getValue().endAddress),
- e.getValue().countryCode,
+ e.getValue().code,
convertKeyToDateString(e.getKey()),
convertDateNumberToString(e.getValue().lastDbDate)));
if (--entries <= 0) {
diff --git a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java
index 77808c8..5f8573a 100644
--- a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java
+++ b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java
@@ -75,11 +75,10 @@ public class DatabasePerformanceExample {
if (testMonth.equals(dbMonth)) {
String testAddressString = DatabaseImpl.
convertKeyToAddressString(test);
- String countryCode =
- temp.lookupCountryCodeFromIpv4AddressAndDate(
+ String code = temp.lookupIpv4AddressAndDate(
testAddressString, testDateString);
- if (countryCode != null) {
- results.put(test, countryCode);
+ if (code != null) {
+ results.put(test, code);
}
}
}
@@ -91,9 +90,9 @@ public class DatabasePerformanceExample {
convertKeyToAddressString(e.getKey());
String testDateString = DatabaseImpl.
convertKeyToDateString(e.getKey());
- String countryCode = e.getValue();
- bw.write(testAddressString + "," + testDateString + ","
- + countryCode + "\n");
+ String code = e.getValue();
+ bw.write(testAddressString + "," + testDateString + "," + code
+ + "\n");
}
bw.close();
long endMillis = System.currentTimeMillis();
@@ -124,7 +123,7 @@ public class DatabasePerformanceExample {
}
String expected = "??".equals(parts[2]) ? null : parts[2];
String result =
- combinedDatabase.lookupCountryCodeFromIpv4AddressAndDate(
+ combinedDatabase.lookupIpv4AddressAndDate(
testAddress, testDate);
tests++;
if ((expected == null && result != null) ||
@@ -165,7 +164,7 @@ public class DatabasePerformanceExample {
String testDate = parts[1];
String expected = parts[2].equals("??") ? null : parts[2];
String result =
- combinedDatabase.lookupCountryCodeFromIpv4AddressAndDate(
+ combinedDatabase.lookupIpv4AddressAndDate(
testAddress, testDate);
tests++;
if ((expected == null && result != null) ||
diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseTest.java b/task-6471/java/src/org/torproject/task6471/DatabaseTest.java
index daba414..fb0f19d 100644
--- a/task-6471/java/src/org/torproject/task6471/DatabaseTest.java
+++ b/task-6471/java/src/org/torproject/task6471/DatabaseTest.java
@@ -15,48 +15,45 @@ public class DatabaseTest {
database.addRange("20120901", "us", "3.0.0.0", 16777216);
database.repairTree();
assertEquals(1, database.getNumberOfElements());
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"2.255.255.255", "19920901"));
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"2.255.255.255", "20020901"));
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"2.255.255.255", "20120901"));
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"2.255.255.255", "20220901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.0.0.0", "19920901"));
- assertEquals("United States",
- database.lookupCountryNameFromIpv4AddressAndDate(
- "3.0.0.0", "19920901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.0.0.0", "20020901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.0.0.0", "20120901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.0.0.0", "20220901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "19920901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20020901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20220901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.255.255.255", "19920901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.255.255.255", "20020901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.255.255.255", "20120901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.255.255.255", "20220901"));
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"4.0.0.0", "19920901"));
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"4.0.0.0", "20020901"));
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"4.0.0.0", "20120901"));
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"4.0.0.0", "20220901"));
}
@@ -67,21 +64,21 @@ public class DatabaseTest {
database.addRange("20120901", "ca", "4.0.0.0", 16777216);
database.repairTree();
assertEquals(2, database.getNumberOfElements());
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"2.255.255.255", "20120901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120901"));
- assertEquals("ca", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("ca", database.lookupIpv4AddressAndDate(
"4.127.0.0", "20120901"));
- assertEquals("ca", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("ca", database.lookupIpv4AddressAndDate(
"4.127.0.0", "20120901"));
- assertEquals("ca", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("ca", database.lookupIpv4AddressAndDate(
"4.127.0.0", "20120901"));
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"5.0.0.0", "20120901"));
}
@@ -92,15 +89,15 @@ public class DatabaseTest {
database.addRange("20120901", "ca", "6.0.0.0", 16777216);
database.repairTree();
assertEquals(2, database.getNumberOfElements());
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"2.255.255.255", "20120901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120901"));
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"4.255.255.255", "20120901"));
- assertEquals("ca", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("ca", database.lookupIpv4AddressAndDate(
"6.127.0.0", "20120901"));
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"7.0.0.0", "20120901"));
}
@@ -111,22 +108,22 @@ public class DatabaseTest {
database.addRange("20120901", "us", "3.0.0.0", 16777216);
database.repairTree();
assertEquals(1, database.getNumberOfElements());
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"2.255.255.255", "20120901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120901"));
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"4.0.0.0", "20120901"));
}
@Test()
- public void testDuplicateImportDifferentCountryCode() {
+ public void testDuplicateImportDifferentCode() {
DatabaseImporterImpl database = new DatabaseImporterImpl();
database.addRange("20120901", "us", "3.0.0.0", 16777216);
database.addRange("20120901", "ca", "3.0.0.0", 16777216);
database.repairTree();
assertEquals(1, database.getNumberOfElements());
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120901"));
}
@@ -138,13 +135,13 @@ public class DatabaseTest {
database.addRange("20121001", "us", "3.0.0.0", 16777216);
database.repairTree();
assertEquals(1, database.getNumberOfElements());
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120801"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20121001"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20121101"));
}
@@ -156,13 +153,13 @@ public class DatabaseTest {
database.addRange("20120901", "us", "3.0.0.0", 16777216);
database.repairTree();
assertEquals(1, database.getNumberOfElements());
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120801"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120901"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20121001"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20121101"));
}
@@ -176,13 +173,13 @@ public class DatabaseTest {
database.addRange("20121001", "us", "6.0.0.0", 16777216);
database.repairTree();
assertEquals(3, database.getNumberOfElements());
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120801"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20120901"));
- assertEquals(null, database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals(null, database.lookupIpv4AddressAndDate(
"3.127.0.0", "20121001"));
- assertEquals("us", database.lookupCountryCodeFromIpv4AddressAndDate(
+ assertEquals("us", database.lookupIpv4AddressAndDate(
"3.127.0.0", "20121101"));
}
}
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits