[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Move probe_ip singleton to the geoip module.
commit 10a802e72c955d450a6a1b56fcb71b4e60d46934
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date: Wed Jul 27 14:24:51 2016 +0200
Move probe_ip singleton to the geoip module.
Make it cache the probe_ip for 10 minutes and support waiting if another
probe_ip lookup is in progress already.
Put a probe ip lookup also inside of the deck
---
ooni/agent/scheduler.py | 25 +++++-----------------
ooni/deck.py | 8 ++++++--
ooni/director.py | 7 +++++--
ooni/geoip.py | 33 ++++++++++++++++++++++++++++--
ooni/nettest.py | 9 ++++----
ooni/nettests/blocking/web_connectivity.py | 8 ++++----
ooni/resources.py | 5 ++++-
ooni/scripts/oonideckgen.py | 3 +--
ooni/settings.py | 3 +--
ooni/templates/httpt.py | 5 +++--
ooni/templates/process.py | 7 ++++---
ooni/tests/test_geoip.py | 2 +-
ooni/ui/cli.py | 2 +-
ooni/ui/web/client/index.html | 2 +-
ooni/ui/web/server.py | 25 ++++++++++++++--------
15 files changed, 89 insertions(+), 55 deletions(-)
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 3a9da20..9784f8a 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -9,6 +9,7 @@ from ooni.utils import log
from ooni.deck import input_store
from ooni.settings import config
from ooni.contrib import croniter
+from ooni.geoip import probe_ip
class ScheduledTask(object):
_time_format = "%Y-%m-%dT%H:%M:%SZ"
@@ -73,19 +74,9 @@ class UpdateInputsAndResources(ScheduledTask):
@defer.inlineCallbacks
def task(self):
log.debug("Updating the inputs")
- yield resources.check_for_update(config.probe_ip.geodata['countrycode'])
- yield input_store.update(config.probe_ip.geodata['countrycode'])
-
-class UpdateProbeIp(ScheduledTask):
- identifier = "ooni-update-probe-ip"
- schedule = "@hourly"
- # XXX we need to ensure this is always run the first time ooniprobe or
- # ooniprobe-agent is started or implement on disk caching of the users
- # IP address.
-
- def task(self):
- log.debug("Updating the probe IP")
- return config.probe_ip.lookup()
+ yield probe_ip.lookup()
+ yield resources.check_for_update(probe_ip.geodata['countrycode'])
+ yield input_store.update(probe_ip.geodata['countrycode'])
class CleanupInProgressReports(ScheduledTask):
identifier = 'ooni-cleanup-reports'
@@ -97,18 +88,13 @@ class UploadMissingReports(ScheduledTask):
# Order mattters
SYSTEM_TASKS = [
- UpdateProbeIp,
UpdateInputsAndResources
]
@defer.inlineCallbacks
-def run_system_tasks(no_geoip=False, no_input_store=False):
+def run_system_tasks(no_input_store=False):
task_classes = SYSTEM_TASKS[:]
- if no_geoip:
- log.debug("Not updating probe IP")
- task_classes.pop(UpdateProbeIp)
-
if no_input_store:
log.debug("Not updating the inputs")
task_classes.pop(UpdateInputsAndResources)
@@ -157,7 +143,6 @@ class SchedulerService(service.MultiService):
def startService(self):
service.MultiService.startService(self)
- self.schedule(UpdateProbeIp())
self.schedule(UpdateInputsAndResources())
self._looping_call.start(self.interval)
diff --git a/ooni/deck.py b/ooni/deck.py
index 0434d81..7976e5e 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -20,6 +20,7 @@ from ooni.resources import check_for_update
from ooni.settings import config
from ooni.utils import generate_filename
from ooni.utils import log
+from ooni.geoip import probe_ip
from ooni.results import generate_summary
@@ -652,7 +653,9 @@ class DeckTask(object):
self.ooni['net_test_loader'] = net_test_loader
+ @defer.inlineCallbacks
def _setup_ooni(self):
+ yield probe_ip.lookup()
for input_file in self.ooni['net_test_loader'].inputFiles:
file_path = resolve_file_path(input_file['filename'], self.cwd)
input_file['test_options'][input_file['key']] = file_path
@@ -660,7 +663,7 @@ class DeckTask(object):
self.id = generate_filename(self.ooni['test_details'])
def setup(self):
- getattr(self, "_setup_"+self.type)()
+ return getattr(self, "_setup_"+self.type)()
def _load(self, data):
for key in self._metadata_keys:
@@ -918,12 +921,13 @@ class NGDeck(object):
d.addErrback(self._measurement_failed, task)
return d
+ @defer.inlineCallbacks
def setup(self):
"""
This method needs to be called before you are able to run a deck.
"""
for task in self._tasks:
- task.setup()
+ yield task.setup()
self._is_setup = True
@defer.inlineCallbacks
diff --git a/ooni/director.py b/ooni/director.py
index 1ab076b..ad6d1e2 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -12,6 +12,7 @@ from ooni.nettest import NetTest, getNetTestInformation
from ooni.settings import config
from ooni.nettest import normalizeTestName
from ooni.deck import InputStore
+from ooni.geoip import probe_ip
from ooni.agent.scheduler import run_system_tasks
from ooni.utils.onion import start_tor, connect_to_control_port
@@ -199,11 +200,13 @@ class Director(object):
aux = map(lambda x: x in annotations, ["city", "country", "asn"])
if not all(aux):
log.msg("You should add annotations for the country, city and ASN")
+ else:
+ yield probe_ip.lookup()
+ self.notify(DirectorEvent("success", "Looked up probe IP"))
self.notify(DirectorEvent("success",
"Running system tasks"))
- yield run_system_tasks(no_geoip=no_geoip,
- no_input_store=not create_input_store)
+ yield run_system_tasks(no_input_store=not create_input_store)
self.notify(DirectorEvent("success",
"Ran system tasks"))
diff --git a/ooni/geoip.py b/ooni/geoip.py
index 2a7ec92..f271790 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -2,6 +2,7 @@ from __future__ import absolute_import
import re
import os
import json
+import time
import random
from hashlib import sha256
@@ -28,7 +29,7 @@ except ImportError:
class GeoIPDataFilesNotFound(Exception):
pass
-def IPToLocation(ipaddr):
+def ip_to_location(ipaddr):
from ooni.settings import config
country_file = config.get_data_file_path(
@@ -152,9 +153,14 @@ class DuckDuckGoGeoIP(HTTPGeoIPLookupper):
probe_ip = re.search(regexp, j['Answer']).group(1)
return probe_ip
+INITIAL = 0
+IN_PROGRESS = 1
+
class ProbeIP(object):
strategy = None
address = None
+ # How long should we consider geoip results valid?
+ _expire_in = 10*60
def __init__(self):
self.geoIPServices = {
@@ -168,10 +174,23 @@ class ProbeIP(object):
'ip': '127.0.0.1'
}
+ self._last_lookup = 0
+ self._reset_state()
+
+ def _reset_state(self):
+ self._state = INITIAL
+ self._looking_up = defer.Deferred()
+ self._looking_up.addCallback(self._looked_up)
+
+ def _looked_up(self, result):
+ self._last_lookup = time.time()
+ self._reset_state()
+ return result
+
def resolveGeodata(self):
from ooni.settings import config
- self.geodata = IPToLocation(self.address)
+ self.geodata = ip_to_location(self.address)
self.geodata['ip'] = self.address
if not config.privacy.includeasn:
self.geodata['asn'] = 'AS0'
@@ -182,13 +201,20 @@ class ProbeIP(object):
@defer.inlineCallbacks
def lookup(self):
+ if self._state == IN_PROGRESS:
+ yield self._looking_up
+ elif self._last_lookup < time.time() - self._expire_in:
+ self.address = None
+
if self.address:
defer.returnValue(self.address)
else:
+ self._state = IN_PROGRESS
try:
yield self.askTor()
log.msg("Found your IP via Tor")
self.resolveGeodata()
+ self._looking_up.callback(self.address)
defer.returnValue(self.address)
except errors.TorStateNotFound:
log.debug("Tor is not running. Skipping IP lookup via Tor.")
@@ -199,6 +225,7 @@ class ProbeIP(object):
yield self.askGeoIPService()
log.msg("Found your IP via a GeoIP service")
self.resolveGeodata()
+ self._looking_up.callback(self.address)
defer.returnValue(self.address)
except Exception:
log.msg("Unable to lookup the probe IP via GeoIPService")
@@ -241,3 +268,5 @@ class ProbeIP(object):
return d
else:
raise errors.TorStateNotFound
+
+probe_ip = ProbeIP()
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 88e4953..566c391 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -13,6 +13,7 @@ from ooni.tasks import Measurement
from ooni.utils import log, sanitize_options, randomStr
from ooni.utils.net import hasRawSocketPermission
from ooni.settings import config
+from ooni.geoip import probe_ip
from ooni import errors as e
@@ -192,10 +193,10 @@ class NetTestLoader(object):
def getTestDetails(self):
return {
- 'probe_asn': config.probe_ip.geodata['asn'],
- 'probe_cc': config.probe_ip.geodata['countrycode'],
- 'probe_ip': config.probe_ip.geodata['ip'],
- 'probe_city': config.probe_ip.geodata['city'],
+ 'probe_asn': probe_ip.geodata['asn'],
+ 'probe_cc': probe_ip.geodata['countrycode'],
+ 'probe_ip': probe_ip.geodata['ip'],
+ 'probe_city': probe_ip.geodata['city'],
'software_name': 'ooniprobe',
'software_version': ooniprobe_version,
# XXX only sanitize the input files
diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index dde6b6f..d8d539f 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -350,10 +350,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
if len(control_addrs.intersection(experiment_addrs)) > 0:
return True
- experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'],
- experiment_addrs))
- control_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'],
- control_addrs))
+ experiment_asns = set(map(lambda x: geoip.ip_to_location(x)['asn'],
+ experiment_addrs))
+ control_asns = set(map(lambda x: geoip.ip_to_location(x)['asn'],
+ control_addrs))
# Remove the instance of AS0 when we fail to find the ASN
control_asns.discard('AS0')
diff --git a/ooni/resources.py b/ooni/resources.py
index 8a0a57f..47ebf86 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -4,7 +4,10 @@ import shutil
from twisted.python.filepath import FilePath
from twisted.internet import defer
-from twisted.web.client import downloadPage, getPage
+from twisted.web.client import downloadPage, getPage, HTTPClientFactory
+
+# Disable logs of HTTPClientFactory
+HTTPClientFactory.noisy = False
from ooni.utils import log, gunzip, rename
from ooni.settings import config
diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py
index 10f8673..9b087f9 100644
--- a/ooni/scripts/oonideckgen.py
+++ b/ooni/scripts/oonideckgen.py
@@ -10,7 +10,7 @@ from twisted.python import usage
from ooni.otime import prettyDateNowUTC
from ooni import errors
-from ooni.geoip import ProbeIP
+from ooni.geoip import probe_ip
from ooni.resources import check_for_update
from ooni.settings import config
from ooni.deck import NGDeck
@@ -86,7 +86,6 @@ def generate_deck(options):
@defer.inlineCallbacks
def get_user_country_code():
config.privacy.includecountry = True
- probe_ip = ProbeIP()
yield probe_ip.lookup()
defer.returnValue(probe_ip.geodata['countrycode'])
diff --git a/ooni/settings.py b/ooni/settings.py
index 36670ac..acb1895 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -23,8 +23,7 @@ class OConfig(object):
self.reports = Storage()
self.scapyFactory = None
self.tor_state = None
- # This is used to store the probes IP address obtained via Tor
- self.probe_ip = geoip.ProbeIP()
+
self.logging = True
self.basic = Storage()
self.advanced = Storage()
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 857392e..cba8702 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -19,6 +19,7 @@ from ooni.common.txextra import TrueHeaders
from ooni.common.txextra import FixedRedirectAgent, TrueHeadersAgent
from ooni.common.http_utils import representBody
from ooni.errors import handleAllFailures
+from ooni.geoip import probe_ip
class InvalidSocksProxyOption(Exception):
pass
@@ -159,9 +160,9 @@ class HTTPTest(NetTestCase):
else:
response_body = ''
# Attempt to redact the IP address of the probe from the responses
- if (config.privacy.includeip is False and config.probe_ip.address is not None and
+ if (config.privacy.includeip is False and probe_ip.address is not None and
(isinstance(response_body, str) or isinstance(response_body, unicode))):
- response_body = response_body.replace(config.probe_ip.address, "[REDACTED]")
+ response_body = response_body.replace(probe_ip.address, "[REDACTED]")
if (getattr(response, 'request', None) and
getattr(response.request, 'absoluteURI', None)):
session['request']['url'] = response.request.absoluteURI
diff --git a/ooni/templates/process.py b/ooni/templates/process.py
index faf0a66..56fe0fd 100644
--- a/ooni/templates/process.py
+++ b/ooni/templates/process.py
@@ -3,6 +3,7 @@ from twisted.internet import protocol, defer, reactor
from ooni.settings import config
from ooni.nettest import NetTestCase
from ooni.utils import log
+from ooni.geoip import probe_ip
class ProcessDirector(protocol.ProcessProtocol):
@@ -108,9 +109,9 @@ class ProcessTest(NetTestCase):
self.report['commands'] = []
# Attempt to redact the IP address of the probe from the standard output
- if config.privacy.includeip is False and config.probe_ip.address is not None:
- result['stdout'] = result['stdout'].replace(config.probe_ip.address, "[REDACTED]")
- result['stderr'] = result['stderr'].replace(config.probe_ip.address, "[REDACTED]")
+ if config.privacy.includeip is False and probe_ip.address is not None:
+ result['stdout'] = result['stdout'].replace(probe_ip.address, "[REDACTED]")
+ result['stderr'] = result['stderr'].replace(probe_ip.address, "[REDACTED]")
self.report['commands'].append({
'command_name': ' '.join(command),
diff --git a/ooni/tests/test_geoip.py b/ooni/tests/test_geoip.py
index 66ba13e..8eb964d 100644
--- a/ooni/tests/test_geoip.py
+++ b/ooni/tests/test_geoip.py
@@ -7,7 +7,7 @@ from ooni import geoip
class TestGeoIP(bases.ConfigTestCase):
def test_ip_to_location(self):
- location = geoip.IPToLocation('8.8.8.8')
+ location = geoip.ip_to_location('8.8.8.8')
assert 'countrycode' in location
assert 'asn' in location
assert 'city' in location
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index d8a4a8f..51ed3b9 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -313,7 +313,7 @@ def runTestWithDirector(director, global_options, url=None,
@defer.inlineCallbacks
def post_director_start(_):
try:
- deck.setup()
+ yield deck.setup()
yield deck.run(director)
except errors.UnableToLoadDeckInput as error:
raise defer.failure.Failure(error)
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index 7859216..c6d83b7 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
<app>
Loading...
</app>
- <script type="text/javascript" src="app.bundle.js?de2d27ce59f4cee8dd96"></script></body>
+ <script type="text/javascript" src="app.bundle.js?16bac0b4c21c5b120b04"></script></body>
</html>
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 0a3d1ca..f14f6b8 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -18,6 +18,7 @@ from ooni.settings import config
from ooni.utils import log
from ooni.director import DirectorEvent
from ooni.results import generate_summary
+from ooni.geoip import probe_ip
config.advanced.debug = True
@@ -82,8 +83,8 @@ class WebUIAPI(object):
self.status = {
"software_version": ooniprobe_version,
"software_name": "ooniprobe",
- "asn": config.probe_ip.geodata['asn'],
- "country_code": config.probe_ip.geodata['countrycode'],
+ "asn": probe_ip.geodata['asn'],
+ "country_code": probe_ip.geodata['countrycode'],
"director_started": False
}
@@ -108,8 +109,8 @@ class WebUIAPI(object):
def director_started(self, _):
self.status['director_started'] = True
- self.status["asn"] = config.probe_ip.geodata['asn']
- self.status["country_code"] = config.probe_ip.geodata['countrycode']
+ self.status["asn"] = probe_ip.geodata['asn']
+ self.status["country_code"] = probe_ip.geodata['countrycode']
@app.handle_errors(NotFound)
def not_found(self, request, _):
@@ -168,10 +169,15 @@ class WebUIAPI(object):
return self.render_json({"command": "deck-list"}, request)
+ @defer.inlineCallbacks
def run_deck(self, deck):
- deck.setup()
- # Here there is a dangling deferred
- deck.run(self.director)
+ # These are dangling deferreds
+ try:
+ yield deck.setup()
+ yield deck.run(self.director)
+ except:
+ self.director_event_poller.notify(DirectorEvent("error",
+ "Failed to start deck"))
@app.route('/api/nettest/<string:test_name>/start', methods=["POST"])
def api_nettest_start(self, request, test_name):
@@ -219,7 +225,10 @@ class WebUIAPI(object):
@app.route('/api/input', methods=["GET"])
def api_input_list(self, request):
- return self.render_json(self.director.input_store.list(), request)
+ input_store_list = self.director.input_store.list()
+ for key, value in input_store_list.items():
+ value.pop('filepath')
+ return self.render_json(input_store_list, request)
@app.route('/api/input/<string:input_id>/content', methods=["GET"])
def api_input_content(self, request, input_id):
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits