[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [onionoo/release] Update to the GeoLite2 ASN database format.
commit 54e26d8a4f109196ebbb5849df05e799b98e63ed
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed Oct 31 09:33:04 2018 +0100
Update to the GeoLite2 ASN database format.
Fixes #28254.
---
CHANGELOG.md | 3 ++
.../metrics/onionoo/updater/LookupService.java | 51 ++++++++++++----------
2 files changed, 32 insertions(+), 22 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c7fc775..3b7dd16 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,8 @@
# Changes in version 7.?-1.??.? - 2018-1?-??
+ * Medium changes
+ - Update to the GeoLite2 ASN database format.
+
* Minor changes
- Rename root package org.torproject.onionoo to
org.torproject.metrics.onionoo.
diff --git a/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java b/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java
index edcb2bd..0967bf2 100644
--- a/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java
+++ b/src/main/java/org/torproject/metrics/onionoo/updater/LookupService.java
@@ -38,7 +38,7 @@ public class LookupService {
private File geoLite2CityLocationsEnCsvFile;
- private File geoIpAsNum2CsvFile;
+ private File geoLite2AsnBlocksIpv4CsvFile;
private boolean hasAllFiles = false;
@@ -62,9 +62,10 @@ public class LookupService {
+ "geoip/.");
return;
}
- this.geoIpAsNum2CsvFile = new File(this.geoipDir, "GeoIPASNum2.csv");
- if (!this.geoIpAsNum2CsvFile.exists()) {
- log.error("No GeoIPASNum2.csv file in geoip/.");
+ this.geoLite2AsnBlocksIpv4CsvFile = new File(this.geoipDir,
+ "GeoLite2-ASN-Blocks-IPv4.csv");
+ if (!this.geoLite2AsnBlocksIpv4CsvFile.exists()) {
+ log.error("No GeoLite2-ASN-Blocks-IPv4.csv file in geoip/.");
return;
}
this.hasAllFiles = true;
@@ -216,29 +217,37 @@ public class LookupService {
}
/* Obtain a map from IP address numbers to ASN. */
- Map<Long, String> addressNumberAsn = new HashMap<>();
+ Map<Long, String[]> addressNumberAsn = new HashMap<>();
try (BufferedReader br = this.createBufferedReaderFromIso88591File(
- this.geoIpAsNum2CsvFile)) {
+ this.geoLite2AsnBlocksIpv4CsvFile)) {
SortedSet<Long> sortedAddressNumbers = new TreeSet<>(
addressStringNumbers.values());
long firstAddressNumber = sortedAddressNumbers.first();
String line;
- long previousStartIpNum = -1L;
+ br.readLine();
while ((line = br.readLine()) != null) {
String[] parts = line.replaceAll("\"", "").split(",", 3);
if (parts.length != 3) {
log.error("Illegal line '{}' in {}.", line,
- geoIpAsNum2CsvFile.getAbsolutePath());
+ this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath());
return lookupResults;
}
try {
- long startIpNum = Long.parseLong(parts[0]);
- if (startIpNum <= previousStartIpNum) {
- log.error("Line '{}' not sorted in {}.", line,
- geoIpAsNum2CsvFile.getAbsolutePath());
+ String[] networkAddressAndMask = parts[0].split("/");
+ String startAddressString = networkAddressAndMask[0];
+ long startIpNum = this.parseAddressString(startAddressString);
+ if (startIpNum < 0L) {
+ log.error("Illegal IP address in '{}' in {}.", line,
+ this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath());
+ return lookupResults;
+ }
+ int networkMaskLength = networkAddressAndMask.length < 2 ? 0
+ : Integer.parseInt(networkAddressAndMask[1]);
+ if (networkMaskLength < 8 || networkMaskLength > 32) {
+ log.error("Missing or illegal network mask in '{}' in {}.", line,
+ this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath());
return lookupResults;
}
- previousStartIpNum = startIpNum;
while (firstAddressNumber < startIpNum
&& firstAddressNumber != -1L) {
sortedAddressNumbers.remove(firstAddressNumber);
@@ -248,12 +257,11 @@ public class LookupService {
firstAddressNumber = sortedAddressNumbers.first();
}
}
- long endIpNum = Long.parseLong(parts[1]);
+ long endIpNum = startIpNum + (1 << (32 - networkMaskLength)) - 1;
while (firstAddressNumber <= endIpNum
&& firstAddressNumber != -1L) {
- if (parts[2].startsWith("AS")) {
- addressNumberAsn.put(firstAddressNumber, parts[2]);
- }
+ addressNumberAsn.put(firstAddressNumber,
+ new String[] { "AS" + parts[1], parts[2] });
sortedAddressNumbers.remove(firstAddressNumber);
if (sortedAddressNumbers.isEmpty()) {
firstAddressNumber = -1L;
@@ -266,13 +274,13 @@ public class LookupService {
}
} catch (NumberFormatException e) {
log.error("Number format exception while parsing line '{}' in {}.",
- line, geoIpAsNum2CsvFile.getAbsolutePath());
+ line, this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath());
return lookupResults;
}
}
} catch (IOException e) {
log.error("I/O exception while reading {}: {}",
- geoIpAsNum2CsvFile.getAbsolutePath(), e);
+ this.geoLite2AsnBlocksIpv4CsvFile.getAbsolutePath(), e);
return lookupResults;
}
@@ -313,10 +321,9 @@ public class LookupService {
lookupResult.setLongitude(latLong[1]);
}
if (addressNumberAsn.containsKey(addressNumber)) {
- String[] parts = addressNumberAsn.get(addressNumber).split(" ",
- 2);
+ String[] parts = addressNumberAsn.get(addressNumber);
lookupResult.setAsNumber(parts[0]);
- lookupResult.setAsName(parts.length == 2 ? parts[1] : "");
+ lookupResult.setAsName(parts[1]);
}
lookupResults.put(addressString, lookupResult);
}
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits