[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Move geoip related classes into ooni.geoip
commit 51fe88e01de3d559522dc2cf3ea518f65993635b
Author: Arturo Filastò <art@xxxxxxxxx>
Date: Fri Mar 15 09:22:13 2013 +0100
Move geoip related classes into ooni.geoip
---
HACKING | 2 +-
ooni/director.py | 133 +--------------------------------------
ooni/geoip.py | 168 +++++++++++++++++++++++++++++++++++++++++++++++++
ooni/nettest.py | 5 +-
ooni/reporter.py | 2 +-
ooni/utils/geodata.py | 40 ------------
tests/test_nettest.py | 2 +-
7 files changed, 176 insertions(+), 176 deletions(-)
diff --git a/HACKING b/HACKING
index b92ddbe..aad7ebb 100644
--- a/HACKING
+++ b/HACKING
@@ -87,7 +87,7 @@ Code Structure
- ooni/utils/
In here go internal utilities that are useful to ooniprobe
-- ooni/utils/geodata.py
+- ooni/geoip.py
In here go functions related to the understanding of
geographical information of the probe
diff --git a/ooni/director.py b/ooni/director.py
index 07510eb..d2f5a7c 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -5,6 +5,7 @@ import os
import re
from ooni import config
+from ooni import geoip
from ooni.managers import ReportEntryManager, MeasurementManager
from ooni.reporter import Report
from ooni.utils import log, checkForRoot
@@ -16,136 +17,6 @@ from txtorcon import TorConfig
from txtorcon import TorState, launch_tor
from twisted.internet import defer, reactor
-from twisted.web import client, http_headers
-from ooni.utils.net import userAgents, BodyReceiver
-
-class HTTPGeoIPLookupper(object):
- url = None
-
- def _response(self, response):
- content_length = response.headers.getRawHeaders('content-length')
-
- finished = defer.Deferred()
- response.deliverBody(BodyReceiver(finished, content_length))
- finished.addCallback(self.parseResponse)
- return finished
-
- def parseResponse(self, response_body):
- """
- Override this with the logic for parsing the response.
-
- Should return the IP address of the probe.
- """
- pass
-
- def failed(self, failure):
- log.err("Failed to lookup via %s" % url)
- log.exception(failure)
- return failure
-
- def lookup(self):
- agent = client.Agent(reactor)
- headers = {}
- headers['User-Agent'] = [random.choice(userAgents)]
-
- d = agent.request("GET", self.url, http_headers.Headers(headers))
- d.addCallback(self._response)
- d.addErrback(self.failed)
- return d
-
-class UbuntuGeoIP(HTTPGeoIPLookupper):
- url = "http://geoip.ubuntu.com/lookup";
-
- def parseResponse(self, response_body):
- response = ET.fromstring(response_body)
- probe_ip = response.find('Ip').text
- return probe_ip
-
-class TorProjectGeoIP(HTTPGeoIPLookupper):
- url = "https://check.torproject.org/";
-
- def parseResponse(self, response_body):
- regexp = "Your IP address appears to be: <b>((\d+\.)+(\d+))"
- probe_ip = re.search(regexp, response_body).group(1)
- return probe_ip
-
-class MaxMindGeoIP(HTTPGeoIPLookupper):
- url = "https://www.maxmind.com/en/locate_my_ip";
-
- def parseResponse(self, response_body):
- regexp = '<span id="my-ip-address">((\d+\.)+(\d+))</span>'
- probe_ip = re.search(regexp, response_body).group(1)
- return probe_ip
-
-class ProbeIP(object):
- strategy = None
- geoIPServices = {'ubuntu': UbuntuGeoIP,
- 'torproject': TorProjectGeoIP,
- 'maximind': MaxMindGeoIP
- }
- address = None
-
- @defer.inlineCallbacks
- def lookup(self):
- try:
- yield self.askTor()
- defer.returnValue(self.address)
- except errors.TorStateNotFound:
- log.debug("Tor is not running. Skipping IP lookup via Tor.")
- except:
- log.msg("Unable to lookup the probe IP via Tor.")
-
- try:
- yield self.askTraceroute()
- defer.returnValue(self.address)
- except errors.InsufficientPrivileges:
- log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges")
- except:
- log.msg("Unable to lookup the probe IP via traceroute")
-
- try:
- yield self.askGeoIPService()
- defer.returnValue(self.address)
- except Exception, e:
- print e
- log.msg("Unable to lookup the probe IP via GeoIPService")
-
- @defer.inlineCallbacks
- def askGeoIPService(self):
- for service_name, service in self.geoIPServices.items():
- s = TorProjectGeoIP()
- log.msg("Looking up your IP address via %s" % service_name)
- try:
- self.address = yield s.lookup()
- self.strategy = 'geo_ip_service-' + service_name
- break
- except:
- log.msg("Failed to lookup your IP via %s" % service_name)
-
- def askTraceroute(self):
- """
- Perform a UDP traceroute to determine the probes IP address.
- """
- checkForRoot()
- raise NotImplemented
-
- def askTor(self):
- """
- Obtain the probes IP address by asking the Tor Control port via GET INFO
- address.
-
- XXX this lookup method is currently broken when there are cached descriptors or consensus documents
- see: https://trac.torproject.org/projects/tor/ticket/8214
- """
- if config.tor_state:
- d = config.tor_state.protocol.get_info("address")
- @d.addCallback
- def cb(result):
- self.strategy = 'tor_get_info_address'
- self.address = result.values()[0]
- return d
- else:
- raise errors.TorStateNotFound
class Director(object):
"""
@@ -225,7 +96,7 @@ class Director(object):
log.msg("Starting Tor...")
yield self.startTor()
- config.probe_ip = ProbeIP()
+ config.probe_ip = geoip.ProbeIP()
yield config.probe_ip.lookup()
@property
diff --git a/ooni/geoip.py b/ooni/geoip.py
new file mode 100644
index 0000000..0fe8a6b
--- /dev/null
+++ b/ooni/geoip.py
@@ -0,0 +1,168 @@
+import re
+import os
+
+from twisted.web import client, http_headers
+from ooni.utils.net import userAgents, BodyReceiver
+from twisted.internet import reactor, defer, protocol
+
+from ooni.utils import log, net
+from ooni import config
+
+try:
+ import pygeoip
+except ImportError:
+ log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements")
+
+class GeoIPDataFilesNotFound(Exception):
+ pass
+
+def IPToLocation(ipaddr):
+ city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
+ country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
+ asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
+
+ location = {'city': None, 'countrycode': None, 'asn': None}
+ try:
+ city_dat = pygeoip.GeoIP(city_file)
+ location['city'] = city_dat.record_by_addr(ipaddr)['city']
+
+ country_dat = pygeoip.GeoIP(country_file)
+ location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
+
+ asn_dat = pygeoip.GeoIP(asn_file)
+ location['asn'] = asn_dat.org_by_addr(ipaddr)
+
+ except IOError:
+ log.err("Could not find GeoIP data files. Go into data/ "
+ "and run make geoip")
+ raise GeoIPDataFilesNotFound
+
+ return location
+
+class HTTPGeoIPLookupper(object):
+ url = None
+
+ def _response(self, response):
+ content_length = response.headers.getRawHeaders('content-length')
+
+ finished = defer.Deferred()
+ response.deliverBody(BodyReceiver(finished, content_length))
+ finished.addCallback(self.parseResponse)
+ return finished
+
+ def parseResponse(self, response_body):
+ """
+ Override this with the logic for parsing the response.
+
+ Should return the IP address of the probe.
+ """
+ pass
+
+ def failed(self, failure):
+ log.err("Failed to lookup via %s" % url)
+ log.exception(failure)
+ return failure
+
+ def lookup(self):
+ agent = client.Agent(reactor)
+ headers = {}
+ headers['User-Agent'] = [random.choice(userAgents)]
+
+ d = agent.request("GET", self.url, http_headers.Headers(headers))
+ d.addCallback(self._response)
+ d.addErrback(self.failed)
+ return d
+
+class UbuntuGeoIP(HTTPGeoIPLookupper):
+ url = "http://geoip.ubuntu.com/lookup";
+
+ def parseResponse(self, response_body):
+ response = ET.fromstring(response_body)
+ probe_ip = response.find('Ip').text
+ return probe_ip
+
+class TorProjectGeoIP(HTTPGeoIPLookupper):
+ url = "https://check.torproject.org/";
+
+ def parseResponse(self, response_body):
+ regexp = "Your IP address appears to be: <b>((\d+\.)+(\d+))"
+ probe_ip = re.search(regexp, response_body).group(1)
+ return probe_ip
+
+class MaxMindGeoIP(HTTPGeoIPLookupper):
+ url = "https://www.maxmind.com/en/locate_my_ip";
+
+ def parseResponse(self, response_body):
+ regexp = '<span id="my-ip-address">((\d+\.)+(\d+))</span>'
+ probe_ip = re.search(regexp, response_body).group(1)
+ return probe_ip
+
+class ProbeIP(object):
+ strategy = None
+ geoIPServices = {'ubuntu': UbuntuGeoIP,
+ 'torproject': TorProjectGeoIP,
+ 'maximind': MaxMindGeoIP
+ }
+ address = None
+
+ @defer.inlineCallbacks
+ def lookup(self):
+ try:
+ yield self.askTor()
+ defer.returnValue(self.address)
+ except errors.TorStateNotFound:
+ log.debug("Tor is not running. Skipping IP lookup via Tor.")
+ except:
+ log.msg("Unable to lookup the probe IP via Tor.")
+
+ try:
+ yield self.askTraceroute()
+ defer.returnValue(self.address)
+ except errors.InsufficientPrivileges:
+ log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges")
+ except:
+ log.msg("Unable to lookup the probe IP via traceroute")
+
+ try:
+ yield self.askGeoIPService()
+ defer.returnValue(self.address)
+ except Exception, e:
+ print e
+ log.msg("Unable to lookup the probe IP via GeoIPService")
+
+ @defer.inlineCallbacks
+ def askGeoIPService(self):
+ for service_name, service in self.geoIPServices.items():
+ s = TorProjectGeoIP()
+ log.msg("Looking up your IP address via %s" % service_name)
+ try:
+ self.address = yield s.lookup()
+ self.strategy = 'geo_ip_service-' + service_name
+ break
+ except:
+ log.msg("Failed to lookup your IP via %s" % service_name)
+
+ def askTraceroute(self):
+ """
+ Perform a UDP traceroute to determine the probes IP address.
+ """
+ checkForRoot()
+ raise NotImplemented
+
+ def askTor(self):
+ """
+ Obtain the probes IP address by asking the Tor Control port via GET INFO
+ address.
+
+ XXX this lookup method is currently broken when there are cached descriptors or consensus documents
+ see: https://trac.torproject.org/projects/tor/ticket/8214
+ """
+ if config.tor_state:
+ d = config.tor_state.protocol.get_info("address")
+ @d.addCallback
+ def cb(result):
+ self.strategy = 'tor_get_info_address'
+ self.address = result.values()[0]
+ return d
+ else:
+ raise errors.TorStateNotFound
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 67dee9d..bd6bd41 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -5,8 +5,9 @@ from twisted.internet import defer, reactor
from twisted.trial.runner import filenameToModule
from twisted.python import usage, reflect
+from ooni import geoip
from ooni.tasks import Measurement
-from ooni.utils import log, checkForRoot, geodata
+from ooni.utils import log, checkForRoot
from ooni import config
from ooni import otime
@@ -35,7 +36,7 @@ class NetTestLoader(object):
config.privacy.includecountry or \
config.privacy.includecity):
log.msg("We will include some geo data in the report")
- client_geodata = geodata.IPToLocation(config.probe_ip.address)
+ client_geodata = geoip.IPToLocation(config.probe_ip.address)
if config.privacy.includeip:
client_geodata['ip'] = config.probe_ip.address
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 20193c9..fc73b67 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -30,7 +30,7 @@ from ooni.errors import InvalidOONIBCollectorAddress
from ooni.errors import ReportNotCreated, ReportAlreadyClosed
from ooni import otime
-from ooni.utils import geodata, pushFilenameStack
+from ooni.utils import pushFilenameStack
from ooni.utils.net import BodyReceiver, StringProducer, userAgents
from ooni import config
diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py
deleted file mode 100644
index d9883ba..0000000
--- a/ooni/utils/geodata.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import re
-import os
-
-from twisted.web.client import Agent
-from twisted.internet import reactor, defer, protocol
-
-from ooni.utils import log, net
-from ooni import config
-
-try:
- import pygeoip
-except ImportError:
- log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements")
-
-class GeoIPDataFilesNotFound(Exception):
- pass
-
-def IPToLocation(ipaddr):
- city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
- country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
- asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
-
- location = {'city': None, 'countrycode': None, 'asn': None}
- try:
- city_dat = pygeoip.GeoIP(city_file)
- location['city'] = city_dat.record_by_addr(ipaddr)['city']
-
- country_dat = pygeoip.GeoIP(country_file)
- location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
-
- asn_dat = pygeoip.GeoIP(asn_file)
- location['asn'] = asn_dat.org_by_addr(ipaddr)
-
- except IOError:
- log.err("Could not find GeoIP data files. Go into data/ "
- "and run make geoip")
- raise GeoIPDataFilesNotFound
-
- return location
-
diff --git a/tests/test_nettest.py b/tests/test_nettest.py
index 976757f..2d070c3 100644
--- a/tests/test_nettest.py
+++ b/tests/test_nettest.py
@@ -241,7 +241,7 @@ class TestNetTest(unittest.TestCase):
# try:
# net_test = NetTestLoader(StringIO(net_test_root_required),
# dummyArgs)
- # except NotRootError:
+ # except errors.InsufficientPrivileges:
# pass
#def test_create_report_succeed(self):
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits