[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/develop] Make the starting of tests more robust.
commit 1247a80ec046e9c3b132ca6b23f602ca852e6b1a
Author: Arturo Filastò <art@xxxxxxxxx>
Date: Wed Feb 27 18:12:13 2013 +0100
Make the starting of tests more robust.
---
ooni/nettest.py | 11 +++++++----
ooni/oonicli.py | 18 +++++++++++++-----
ooni/utils/geodata.py | 39 +++++++++++++++++++++++++++++++++++++++
3 files changed, 59 insertions(+), 9 deletions(-)
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 7108cd1..0260ca2 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -39,7 +39,11 @@ class NetTestLoader(object):
config.privacy.includecountry or \
config.privacy.includecity):
log.msg("We will include some geo data in the report")
- client_geodata = geoip.IPToLocation(config.probe_ip.address)
+ try:
+ client_geodata = geodata.IPToLocation(config.probe_ip)
+ except e.GeoIPDataFilesNotFound:
+ log.err("Unable to find the geoip data files")
+ client_geodata = {'city': None, 'countrycode': None, 'asn': None}
if config.privacy.includeip:
client_geodata['ip'] = config.probe_ip.address
@@ -52,8 +56,7 @@ class NetTestLoader(object):
client_geodata['asn'] = 'AS0'
elif 'asn' in client_geodata:
# XXX this regexp should probably go inside of geodata
- client_geodata['asn'] = \
- re.search('AS\d+', client_geodata['asn']).group(0)
+ client_geodata['asn'] = client_geodata['asn']
log.msg("Your AS number is: %s" % client_geodata['asn'])
else:
client_geodata['asn'] = None
@@ -307,7 +310,7 @@ class NetTest(object):
self.state.taskDone()
if len(self.report.reporters) == 0:
- raise AllReportersFailed
+ raise e.AllReportersFailed
return report_results
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 73d7709..93bf8bf 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -35,8 +35,8 @@ class Options(usage.Options):
optParameters = [["reportfile", "o", None, "report file name"],
["testdeck", "i", None,
"Specify as input a test deck: a yaml file containig the tests to run an their arguments"],
- ["collector", "c", None,
- "Address of the collector of test results. (example: http://127.0.0.1:8888)"],
+ ["collector", "c", 'httpo://nkvphnp3p6agi5qq.onion',
+ "Address of the collector of test results. default: httpo://nkvphnp3p6agi5qq.onion"],
["logfile", "l", None, "log file name"],
["pcapfile", "O", None, "pcap file name"],
["parallelism", "p", "10", "input parallelism"],
@@ -130,15 +130,22 @@ def runWithDirector():
director = Director()
d = director.start()
+ def director_startup_failed(failure):
+ log.err("Failed to start the director")
+ log.exception(failure)
+ reactor.stop()
+
# Wait until director has started up (including bootstrapping Tor) before adding tess
def post_director_start(_):
for net_test_loader in test_list:
- yaml_reporter = YAMLReporter(net_test_loader.testDetails)
+ test_details = net_test_loader.testDetails
+
+ yaml_reporter = YAMLReporter(test_details)
reporters = [yaml_reporter]
if global_options['collector']:
try:
- oonib_reporter = OONIBReporter(net_test_loader.testDetails,
+ oonib_reporter = OONIBReporter(test_details,
global_options['collector'])
reporters.append(oonib_reporter)
except InvalidOONIBCollectorAddress:
@@ -152,7 +159,7 @@ def runWithDirector():
with open('collector') as f:
reporter_url = random.choice(f.readlines())
reporter_url = reporter_url.split('#')[0].strip()
- oonib_reporter = OONIBReporter(net_test_loader.testDetails, reporter_url)
+ oonib_reporter = OONIBReporter(test_details, reporter_url)
reporters.append(oonib_reporter)
log.debug("adding callback for startNetTest")
@@ -160,4 +167,5 @@ def runWithDirector():
d.addCallback(shutdown)
d.addCallback(post_director_start)
+ d.addErrback(director_startup_failed)
reactor.run()
diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py
new file mode 100644
index 0000000..56ce05c
--- /dev/null
+++ b/ooni/utils/geodata.py
@@ -0,0 +1,39 @@
+import re
+import os
+
+from twisted.web.client import Agent
+from twisted.internet import reactor, defer, protocol
+
+from ooni.utils import log, net
+from ooni import config
+from ooni.errors import GeoIPDataFilesNotFound
+
+try:
+ import pygeoip
+except ImportError:
+ log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements")
+
+def IPToLocation(ipaddr):
+ city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
+ country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
+ asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
+
+ location = {'city': None, 'countrycode': None, 'asn': None}
+ try:
+ city_dat = pygeoip.GeoIP(city_file)
+ location['city'] = city_dat.record_by_addr(ipaddr)['city']
+
+ country_dat = pygeoip.GeoIP(country_file)
+ location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
+
+ asn_dat = pygeoip.GeoIP(asn_file)
+ asn = asn_dat.org_by_addr(ipaddr)
+ location['asn'] = re.search('AS\d+', asn).group(0)
+
+ except IOError:
+ log.err("Could not find GeoIP data files. Go into data/ "
+ "and run make geoip")
+ raise GeoIPDataFilesNotFound
+
+ return location
+
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits