[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [onionoo/master] Fix character encoding when reading GeoIP files.
commit 5fe615f50a251984ce9e95f37428edb152b08d6c
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Mon Feb 16 16:25:12 2015 +0100
Fix character encoding when reading GeoIP files.
Found while testing special characters as suggested by iwakeh.
---
.../torproject/onionoo/updater/LookupService.java | 6 +--
.../org/torproject/onionoo/LookupServiceTest.java | 55 ++++++++++++++++++--
2 files changed, 55 insertions(+), 6 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/updater/LookupService.java b/src/main/java/org/torproject/onionoo/updater/LookupService.java
index 85e29f4..f88e752 100644
--- a/src/main/java/org/torproject/onionoo/updater/LookupService.java
+++ b/src/main/java/org/torproject/onionoo/updater/LookupService.java
@@ -116,7 +116,7 @@ public class LookupService {
addressStringNumbers.values());
BufferedReader br = new BufferedReader(new InputStreamReader(
new FileInputStream(this.geoLite2CityBlocksIPv4CsvFile),
- "ISO-8859-1"));
+ "UTF-8"));
String line = br.readLine();
while ((line = br.readLine()) != null) {
String[] parts = line.split(",", 9);
@@ -188,7 +188,7 @@ public class LookupService {
addressNumberBlocks.values());
BufferedReader br = new BufferedReader(new InputStreamReader(
new FileInputStream(this.geoLite2CityLocationsEnCsvFile),
- "ISO-8859-1"));
+ "UTF-8"));
String line = br.readLine();
while ((line = br.readLine()) != null) {
String[] parts = line.replaceAll("\"", "").split(",", 13);
@@ -228,7 +228,7 @@ public class LookupService {
addressStringNumbers.values());
long firstAddressNumber = sortedAddressNumbers.first();
BufferedReader br = new BufferedReader(new InputStreamReader(
- new FileInputStream(geoIPASNum2CsvFile), "ISO-8859-1"));
+ new FileInputStream(geoIPASNum2CsvFile), "UTF-8"));
String line;
long previousStartIpNum = -1L;
while ((line = br.readLine()) != null) {
diff --git a/src/test/java/org/torproject/onionoo/LookupServiceTest.java b/src/test/java/org/torproject/onionoo/LookupServiceTest.java
index 008b21c..7d93b04 100644
--- a/src/test/java/org/torproject/onionoo/LookupServiceTest.java
+++ b/src/test/java/org/torproject/onionoo/LookupServiceTest.java
@@ -8,9 +8,10 @@ import static org.junit.Assert.assertTrue;
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.List;
@@ -83,8 +84,9 @@ public class LookupServiceTest {
private void writeCsvFile(List<String> lines, String fileName)
throws IOException {
if (lines != null && !lines.isEmpty()) {
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- new File(this.tempGeoipDir, fileName)));
+ BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
+ new FileOutputStream(new File(this.tempGeoipDir, fileName)),
+ "UTF-8"));
for (String line : lines) {
bw.write(line + "\n");
}
@@ -381,5 +383,52 @@ public class LookupServiceTest {
this.assertLookupResult(null, null, geoipASNum2Lines, "8.8.8.8", null,
null, null, null, null, null, null, null);
}
+
+ @Test()
+ public void testLookupLocationSpecialCharacters() {
+ List<String> geoLite2CityBlocksIPv4Lines = new ArrayList<String>();
+ geoLite2CityBlocksIPv4Lines.add("network,geoname_id,"
+ + "registered_country_geoname_id,represented_country_geoname_id,"
+ + "is_anonymous_proxy,is_satellite_provider,postal_code,latitude,"
+ + "longitude");
+ geoLite2CityBlocksIPv4Lines.add("46.1.133.0/24,307515,298795,,0,0,,"
+ + "39.1458,34.1639");
+ geoLite2CityBlocksIPv4Lines.add("46.196.12.0/24,738927,298795,,0,0,,"
+ + "40.9780,27.5085");
+ geoLite2CityBlocksIPv4Lines.add("78.180.14.0/24,745169,298795,,0,0,,"
+ + "40.0781,29.5133");
+ geoLite2CityBlocksIPv4Lines.add("81.215.1.0/24,749748,298795,,0,0,,"
+ + "40.6000,33.6153");
+ List<String> geoLite2CityLocationsEnLines = new ArrayList<String>();
+ geoLite2CityLocationsEnLines.add("geoname_id,locale_code,"
+ + "continent_code,continent_name,country_iso_code,country_name,"
+ + "subdivision_1_iso_code,subdivision_1_name,"
+ + "subdivision_2_iso_code,subdivision_2_name,city_name,"
+ + "metro_code,time_zone");
+ geoLite2CityLocationsEnLines.add("307515,en,AS,Asia,TR,Turkey,40,"
+ + "\"K\u0131r\u015Fehir\",,,\"K\u0131r\u015Fehir\",,"
+ + "Europe/Istanbul");
+ geoLite2CityLocationsEnLines.add("738927,en,AS,Asia,TR,Turkey,59,"
+ + "\"Tekirda\u011F\",,,\"Tekirda\u011F\",,Europe/Istanbul");
+ geoLite2CityLocationsEnLines.add("745169,en,AS,Asia,TR,Turkey,16,"
+ + "Bursa,,,\u0130neg\u00F6l,,Europe/Istanbul");
+ geoLite2CityLocationsEnLines.add("749748,en,AS,Asia,TR,Turkey,18,"
+ + "\"\u00C7ank\u0131r\u0131\",,,\"\u00C7ank\u0131r\u0131\",,"
+ + "Europe/Istanbul");
+ this.assertLookupResult(geoLite2CityBlocksIPv4Lines,
+ geoLite2CityLocationsEnLines, null, "46.1.133.0", "tr", "Turkey",
+ "K\u0131r\u015Fehir", "K\u0131r\u015Fehir", 39.1458f, 34.1639f,
+ null, null);
+ this.assertLookupResult(geoLite2CityBlocksIPv4Lines,
+ geoLite2CityLocationsEnLines, null, "46.196.12.0", "tr", "Turkey",
+ "Tekirda\u011F", "Tekirda\u011F", 40.9780f, 27.5085f, null, null);
+ this.assertLookupResult(geoLite2CityBlocksIPv4Lines,
+ geoLite2CityLocationsEnLines, null, "78.180.14.0", "tr", "Turkey",
+ "Bursa", "\u0130neg\u00F6l", 40.0781f, 29.5133f, null, null);
+ this.assertLookupResult(geoLite2CityBlocksIPv4Lines,
+ geoLite2CityLocationsEnLines, null, "81.215.1.0", "tr", "Turkey",
+ "\u00C7ank\u0131r\u0131", "\u00C7ank\u0131r\u0131", 40.6000f,
+ 33.6153f, null, null);
+ }
}
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits