[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Add support for looking up test helpers via a bouncer.
commit 3bface3bd4fab9e87bdc8c440d70e07e19b8af28
Author: Arturo Filastò <art@xxxxxxxxx>
Date: Fri Aug 23 16:46:15 2013 +0200
Add support for looking up test helpers via a bouncer.
---
data/nettests/blocking/dnsconsistency.py | 4 +-
data/nettests/manipulation/dnsspoof.py | 1 +
.../manipulation/http_header_field_manipulation.py | 1 +
data/nettests/manipulation/http_host.py | 1 +
.../manipulation/http_invalid_request_line.py | 4 +-
data/nettests/manipulation/traceroute.py | 3 +-
decks/basic.deck | 2 +-
ooni/deck.py | 50 +++++++++++++++---
ooni/errors.py | 2 +
ooni/nettest.py | 28 ++++++++--
ooni/oonibclient.py | 55 ++++++++++++++------
ooni/oonicli.py | 42 ++++++++++-----
ooni/reporter.py | 8 ++-
ooni/tests/test_oonibclient.py | 22 ++++++++
ooni/utils/net.py | 9 ++--
15 files changed, 183 insertions(+), 49 deletions(-)
diff --git a/data/nettests/blocking/dnsconsistency.py b/data/nettests/blocking/dnsconsistency.py
index 7b6e7b9..3c88cd2 100644
--- a/data/nettests/blocking/dnsconsistency.py
+++ b/data/nettests/blocking/dnsconsistency.py
@@ -38,12 +38,14 @@ class DNSConsistencyTest(dnst.DNSTest):
name = "DNS Consistency"
description = "DNS censorship detection test"
- version = "0.5"
+ version = "0.6"
authors = "Arturo Filastò, Isis Lovecruft"
requirements = None
inputFile = ['file', 'f', None,
'Input file of list of hostnames to attempt to resolve']
+
+ requiredTestHelpers = {'backend': 'dns'}
usageOptions = UsageOptions
requiredOptions = ['backend', 'file']
diff --git a/data/nettests/manipulation/dnsspoof.py b/data/nettests/manipulation/dnsspoof.py
index 5c50c2f..c9120a4 100644
--- a/data/nettests/manipulation/dnsspoof.py
+++ b/data/nettests/manipulation/dnsspoof.py
@@ -22,6 +22,7 @@ class DNSSpoof(scapyt.ScapyTest):
usageOptions = UsageOptions
+ requiredTestHelpers = {'backend': 'dns'}
requiredOptions = ['hostname', 'resolver']
def setUp(self):
diff --git a/data/nettests/manipulation/http_header_field_manipulation.py b/data/nettests/manipulation/http_header_field_manipulation.py
index 509f4ef..3423442 100644
--- a/data/nettests/manipulation/http_header_field_manipulation.py
+++ b/data/nettests/manipulation/http_header_field_manipulation.py
@@ -48,6 +48,7 @@ class HTTPHeaderFieldManipulation(httpt.HTTPTest):
randomizeUA = False
usageOptions = UsageOptions
+ requiredTestHelpers = {'backend': 'http-return-json-headers'}
requiredOptions = ['backend']
def get_headers(self):
diff --git a/data/nettests/manipulation/http_host.py b/data/nettests/manipulation/http_host.py
index 3f1e0c6..2ec517c 100644
--- a/data/nettests/manipulation/http_host.py
+++ b/data/nettests/manipulation/http_host.py
@@ -43,6 +43,7 @@ class HTTPHost(httpt.HTTPTest):
inputFile = ['file', 'f', None,
'List of hostnames to test for censorship']
+ requiredTestHelpers = {'backend': 'http-return-json-headers'}
requiredOptions = ['backend']
def test_filtering_prepend_newline_to_method(self):
diff --git a/data/nettests/manipulation/http_invalid_request_line.py b/data/nettests/manipulation/http_invalid_request_line.py
index 2482282..64dbcac 100644
--- a/data/nettests/manipulation/http_invalid_request_line.py
+++ b/data/nettests/manipulation/http_invalid_request_line.py
@@ -20,10 +20,12 @@ class HTTPInvalidRequestLine(tcpt.TCPTest):
ascii letters or numbers ('XxXx' will be 4).
"""
name = "HTTP Invalid Request Line"
- version = "0.1.4"
+ version = "0.2"
authors = "Arturo Filastò"
usageOptions = UsageOptions
+
+ requiredTestHelpers = {'backend': 'tcp-echo'}
requiredOptions = ['backend']
def setUp(self):
diff --git a/data/nettests/manipulation/traceroute.py b/data/nettests/manipulation/traceroute.py
index 3f6f17b..2db1826 100644
--- a/data/nettests/manipulation/traceroute.py
+++ b/data/nettests/manipulation/traceroute.py
@@ -23,8 +23,9 @@ class UsageOptions(usage.Options):
class TracerouteTest(scapyt.BaseScapyTest):
name = "Multi Protocol Traceroute Test"
author = "Arturo Filastò"
- version = "0.1.1"
+ version = "0.2"
+ requiredTestHelpers = {'backend': 'traceroute'}
usageOptions = UsageOptions
dst_ports = [0, 22, 23, 53, 80, 123, 443, 8080, 65535]
diff --git a/decks/basic.deck b/decks/basic.deck
index c784cd5..84a8f9d 100644
--- a/decks/basic.deck
+++ b/decks/basic.deck
@@ -7,7 +7,7 @@
pcapfile: null
reportfile: null
resume: 0
- subargs: [-f, 'httpo://3mr5phzltfzqgh6y.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1']
+ subargs: [-f, 'httpo://amcq5ldf3vwg22ze.onion/input/37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1']
test_file: data/nettests/blocking/http_requests.py
testdeck: null
- options:
diff --git a/ooni/deck.py b/ooni/deck.py
index 8f1335d..1ad8377 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -4,7 +4,7 @@ from ooni.nettest import NetTestLoader
from ooni.settings import config
from ooni.utils import log
from ooni.utils.txagentwithsocks import Agent
-from ooni.errors import UnableToLoadDeckInput
+from ooni import errors as e
from ooni.oonibclient import OONIBClient
from twisted.internet import reactor, defer
@@ -14,9 +14,12 @@ import re
import yaml
class Deck(object):
- def __init__(self, deckFile=None):
+ def __init__(self, bouncer, deckFile=None):
+ self.bouncer = bouncer
self.netTestLoaders = []
self.inputs = []
+ self.testHelpers = {}
+ self.collector = None
if deckFile: self.loadDeck(deckFile)
@@ -31,14 +34,20 @@ class Deck(object):
def insert(self, net_test_loader):
""" Add a NetTestLoader to this test deck """
net_test_loader.checkOptions()
- self.fetchAndVerifyNetTestInput(net_test_loader)
self.netTestLoaders.append(net_test_loader)
-
+
+ def getRequiredTestHelpers(self):
+ for net_test_loader in self.netTestLoaders:
+ for test_helper in net_test_loader.requiredTestHelpers:
+ self.testHelpers[test_helper['name']] = None
+
@defer.inlineCallbacks
- def fetchAndVerifyDeckInputs(self):
+ def setup(self):
""" fetch and verify inputs for all NetTests in the deck """
for net_test_loader in self.netTestLoaders:
yield self.fetchAndVerifyNetTestInput(net_test_loader)
+ self.getRequiredTestHelpers()
+ yield self.lookupTestHelpers()
@defer.inlineCallbacks
def fetchAndVerifyNetTestInput(self, net_test_loader):
@@ -47,12 +56,37 @@ class Deck(object):
for i in net_test_loader.inputFiles:
if 'url' in i:
log.debug("Downloading %s" % i['url'])
- oonib = OONIBClient(i['address'])
+ oonibclient = OONIBClient(i['address'])
+
+ try:
+ input_file = yield oonibclient.downloadInput(i['hash'])
+ except:
+ raise e.UnableToLoadDeckInput
- input_file = yield oonib.downloadInput(i['hash'])
try:
input_file.verify()
except AssertionError:
- raise UnableToLoadDeckInput, cached_path
+ raise e.UnableToLoadDeckInput, cached_path
i['test_class'].localOptions[i['key']] = input_file.cached_file
+
+ def setNettestOptions(self):
+ for net_test_loader in self.netTestLoaders:
+ for th in net_test_loader.requiredTestHelpers:
+ test_helper_address = self.testHelpers[th['name']]
+ th['test_class'].localOptions[th['option']] = test_helper_address
+ net_test_loader.collector = self.collector
+ log.debug("Using %s: %s" % (test_helper_address, self.collector))
+
+ @defer.inlineCallbacks
+ def lookupTestHelpers(self):
+ log.msg("Looking up test helpers: %s" % self.testHelpers.keys())
+
+ required_test_helpers = self.testHelpers.keys()
+ if required_test_helpers:
+ oonibclient = OONIBClient(self.bouncer)
+ test_helpers = yield oonibclient.lookupTestHelpers(required_test_helpers)
+ self.collector = test_helpers['collector']
+ for name in self.testHelpers.keys():
+ self.testHelpers[name] = test_helpers[name]
+ self.setNettestOptions()
diff --git a/ooni/errors.py b/ooni/errors.py
index 94a83a4..5ad1940 100644
--- a/ooni/errors.py
+++ b/ooni/errors.py
@@ -170,3 +170,5 @@ class OONIBTestDetailsLookupError(OONIBReportError):
class UnableToLoadDeckInput(Exception):
pass
+class CouldNotFindTestHelper(Exception):
+ pass
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 1699f19..2b170d0 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -172,6 +172,7 @@ def getNetTestInformation(net_test_file):
class NetTestLoader(object):
method_prefix = 'test'
+ collector = None
def __init__(self, options, test_file=None, test_string=None):
self.onionInputRegex = re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]{64})$")
@@ -185,7 +186,22 @@ class NetTestLoader(object):
if test_cases:
self.setupTestCases(test_cases)
-
+
+ @property
+ def requiredTestHelpers(self):
+ required_test_helpers = []
+ if not self.testCases:
+ return required_test_helpers
+
+ for test_class, test_methods in self.testCases:
+ for option, name in test_class.requiredTestHelpers.items():
+ required_test_helpers.append({
+ 'name': name,
+ 'option': option,
+ 'test_class': test_class
+ })
+ return required_test_helpers
+
@property
def inputFiles(self):
input_files = []
@@ -255,6 +271,10 @@ class NetTestLoader(object):
if (client_geodata and not config.privacy.includecountry) \
or ('countrycode' not in client_geodata):
client_geodata['countrycode'] = None
+
+ input_file_hashes = []
+ for input_file in self.inputFiles:
+ input_file_hashes.append(input_file['hash'])
test_details = {'start_time': time.time(),
'probe_asn': client_geodata['asn'],
@@ -264,7 +284,8 @@ class NetTestLoader(object):
'test_version': self.testVersion,
'software_name': 'ooniprobe',
'software_version': software_version,
- 'options': self.options
+ 'options': self.options,
+ 'input_hashes': input_file_hashes
}
return test_details
@@ -613,7 +634,8 @@ class NetTestCase(object):
optParameters = None
baseParameters = None
baseFlags = None
-
+
+ requiredTestHelpers = {}
requiredOptions = []
requiresRoot = False
diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py
index 5861ff4..9bdce95 100644
--- a/ooni/oonibclient.py
+++ b/ooni/oonibclient.py
@@ -7,6 +7,7 @@ from twisted.internet import defer, reactor
from ooni.utils.txagentwithsocks import Agent
+from ooni import errors as e
from ooni.settings import config
from ooni.utils import log
from ooni.utils.net import BodyReceiver, StringProducer, Downloader
@@ -64,35 +65,46 @@ class InputFile(object):
file_hash = sha256(f.read())
assert file_hash.hexdigest() == digest
- def validate(self, policy):
- """
- Validate this input file against the specified input policy.
- """
- for input_file
-
class Collector(object):
- def __init__(self):
+ def __init__(self, address):
+ self.address = address
+
self.nettest_policy = None
self.input_policy = None
+
+ @defer.inlineCallbacks
+ def loadPolicy(self):
+ # XXX implement caching of policies
+ oonibclient = OONIBClient(self.address)
+ log.msg("Looking up nettest policy for %s" % self.address)
+ self.nettest_policy = yield oonibclient.getNettestPolicy()
+ log.msg("Looking up input policy for %s" % self.address)
+ self.input_policy = yield oonibclient.getInputPolicy()
def validateInput(self, input_hash):
- pass
+ for i in self.input_policy:
+ if i['id'] == input_hash:
+ return True
+ return False
- def validateNettest(self, nettest):
- pass
+ def validateNettest(self, nettest_name):
+ for i in self.nettest_policy:
+ if nettest_name == i['name']:
+ return True
+ return False
class OONIBClient(object):
def __init__(self, address):
self.address = address
self.agent = Agent(reactor, sockshost="127.0.0.1",
socksport=config.tor.socks_port)
- self.input_files = {}
def _request(self, method, urn, genReceiver, bodyProducer=None):
finished = defer.Deferred()
uri = self.address + urn
- d = self.agent.request(method, uri, bodyProducer)
+ headers = {}
+ d = self.agent.request(method, uri, bodyProducer=bodyProducer)
@d.addCallback
def callback(response):
@@ -108,7 +120,7 @@ class OONIBClient(object):
def queryBackend(self, method, urn, query=None):
bodyProducer = None
if query:
- bodyProducer = StringProducer(json.dumps(query), bodyProducer)
+ bodyProducer = StringProducer(json.dumps(query))
def genReceiver(finished, content_length):
return BodyReceiver(finished, content_length, json.loads)
@@ -125,9 +137,6 @@ class OONIBClient(object):
def getNettestPolicy(self):
pass
- def queryBouncer(self, requested_helpers):
- pass
-
def getInput(self, input_hash):
input_file = InputFile(input_hash)
if input_file.descriptorCached:
@@ -176,3 +185,17 @@ class OONIBClient(object):
def getNettestPolicy(self):
return self.queryBackend('GET', '/policy/nettest')
+
+ @defer.inlineCallbacks
+ def lookupTestHelpers(self, test_helper_names):
+ try:
+ test_helpers = yield self.queryBackend('POST', '/bouncer',
+ query={'test-helpers': test_helper_names})
+ except Exception:
+ raise e.CouldNotFindTestHelper
+
+ if not test_helpers:
+ raise e.CouldNotFindTestHelper
+
+ defer.returnValue(test_helpers)
+
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index ef6ef84..cf7d34d 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -36,8 +36,10 @@ class Options(usage.Options):
optParameters = [["reportfile", "o", None, "report file name"],
["testdeck", "i", None,
"Specify as input a test deck: a yaml file containig the tests to run an their arguments"],
- ["collector", "c", 'httpo://nkvphnp3p6agi5qq.onion',
- "Address of the collector of test results. default: httpo://nkvphnp3p6agi5qq.onion"],
+ ["collector", "c", None,
+ "Address of the collector of test results. This option should not be used, but you should always use a bouncer."],
+ ["bouncer", "b", 'httpo://nkvphnp3p6agi5qq.onion',
+ "Address of the bouncer for test helpers. default: httpo://nkvphnp3p6agi5qq.onion"],
["logfile", "l", None, "log file name"],
["pcapfile", "O", None, "pcap file name"],
["parallelism", "p", "10", "input parallelism"],
@@ -125,7 +127,7 @@ def runWithDirector():
director = Director()
d = director.start()
- deck = Deck()
+ deck = Deck(global_options['bouncer'])
if global_options['no-collector']:
log.msg("Not reporting using a collector")
collector = global_options['collector'] = None
@@ -146,10 +148,10 @@ def runWithDirector():
log.err(e)
print net_test_loader.usageOptions().getUsage()
sys.exit(2)
-
- def fetch_nettest_inputs(result):
+
+ def setup_nettest(_):
try:
- return deck.fetchAndVerifyDeckInputs()
+ return deck.setup()
except errors.UnableToLoadDeckInput, e:
return defer.failure.Failure(result)
@@ -157,18 +159,30 @@ def runWithDirector():
log.err("Failed to start the director")
r = failure.trap(errors.TorNotRunning,
errors.InvalidOONIBCollectorAddress,
- errors.UnableToLoadDeckInput)
- if r == errors.TorNotRunning:
+ errors.UnableToLoadDeckInput, errors.CouldNotFindTestHelper)
+
+ if isinstance(failure.value, errors.TorNotRunning):
log.err("Tor does not appear to be running")
log.err("Reporting with the collector %s is not possible" %
global_options['collector'])
log.msg("Try with a different collector or disable collector reporting with -n")
- elif r == errors.InvalidOONIBCollectorAddress:
+
+ elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
log.err("Invalid format for oonib collector address.")
log.msg("Should be in the format http://<collector_address>:<port>")
log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
- elif r == errors.UnableToLoadDeckInput:
- log.err('Missing required input files: %s' % failure)
+
+ elif isinstance(failure.value, errors.UnableToLoadDeckInput):
+ log.err("Unable to fetch the required inputs for the test deck.")
+ log.msg("Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues";)
+
+ elif isinstance(failure.value, errors.CouldNotFindTestHelper):
+ log.err("Unable to obtain the required test helpers.")
+ log.msg("Try with a different bouncer or check that Tor is running properly.")
+
+ if config.advanced.debug:
+ log.exception(failure)
+
reactor.stop()
# Wait until director has started up (including bootstrapping Tor)
@@ -188,8 +202,8 @@ def runWithDirector():
if not global_options['no-collector']:
if global_options['collector']:
collector = global_options['collector']
- elif net_test_loader.options['collector']:
- collector = net_test_loader.options['collector']
+ elif net_test_loader.collector:
+ collector = net_test_loader.collector
if collector and collector.startswith('httpo:') \
and (not (config.tor_state or config.tor.socks_port)):
@@ -213,7 +227,7 @@ def runWithDirector():
director.allTestsDone.addBoth(shutdown)
def start():
- d.addCallback(fetch_nettest_inputs)
+ d.addCallback(setup_nettest)
d.addCallback(post_director_start)
d.addErrback(director_startup_failed)
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 0c13cf2..8a89c8a 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -308,6 +308,7 @@ class OONIBReporter(OReporter):
'probe_asn': self.testDetails['probe_asn'],
'test_name': self.testDetails['test_name'],
'test_version': self.testDetails['test_version'],
+ 'input_hashes': self.testDetails['input_hashes'],
# XXX there is a bunch of redundancy in the arguments getting sent
# to the backend. This may need to get changed in the client and the
# backend.
@@ -328,7 +329,6 @@ class OONIBReporter(OReporter):
bodyProducer=bodyProducer)
except ConnectionRefusedError:
log.err("Connection to reporting backend failed (ConnectionRefusedError)")
- #yield defer.fail(OONIBReportCreationError())
raise errors.OONIBReportCreationError
except errors.HostUnreachable:
@@ -353,6 +353,12 @@ class OONIBReporter(OReporter):
log.err("Failed to parse collector response")
log.exception(e)
raise errors.OONIBReportCreationError
+
+ if response.code == 400:
+ # XXX make this more strict
+ log.err("The specified input or nettests cannot be submitted to this collector.")
+ log.msg("Try running a different test or try reporting to a different collector.")
+ raise errors.OONIBReportCreationError
self.reportID = parsed_response['report_id']
self.backendVersion = parsed_response['backend_version']
diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py
index c039c65..1a1c4bc 100644
--- a/ooni/tests/test_oonibclient.py
+++ b/ooni/tests/test_oonibclient.py
@@ -1,10 +1,17 @@
+import os
+import shutil
import socket
from twisted.trial import unittest
from twisted.internet import defer
+from ooni import errors as e
+from ooni.utils import log
+from ooni.settings import config
from ooni.oonibclient import OONIBClient
+data_dir = '/tmp/testooni'
+config.advanced.data_dir = data_dir
input_id = '37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'
class TestOONIBClient(unittest.TestCase):
@@ -15,6 +22,10 @@ class TestOONIBClient(unittest.TestCase):
try:
s.connect((host, port))
s.shutdown(2)
+ try: shutil.rmtree(data_dir)
+ except: pass
+ os.mkdir(data_dir)
+ os.mkdir(os.path.join(data_dir, 'inputs'))
except Exception as ex:
self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)")
self.oonibclient = OONIBClient('http://' + host + ':' + str(port))
@@ -51,6 +62,17 @@ class TestOONIBClient(unittest.TestCase):
def test_download_deck(self):
pass
+ def test_lookup_invalid_helpers(self):
+ return self.failUnlessFailure(
+ self.oonibclient.lookupTestHelpers(
+ ['dns', 'http-return-json-headers', 'sdadsadsa']
+ ), e.CouldNotFindTestHelper)
+
+ @defer.inlineCallbacks
+ def test_lookup_test_helpers(self):
+ helpers = yield self.oonibclient.lookupTestHelpers(['dns', 'http-return-json-headers'])
+ self.assertTrue(len(helpers) == 1)
+
@defer.inlineCallbacks
def test_get_nettest_list(self):
input_list = yield self.oonibclient.getInputList()
diff --git a/ooni/utils/net.py b/ooni/utils/net.py
index 1ec9608..845cf4c 100644
--- a/ooni/utils/net.py
+++ b/ooni/utils/net.py
@@ -77,9 +77,12 @@ class BodyReceiver(protocol.Protocol):
self.bytes_remaining -= len(b)
def connectionLost(self, reason):
- if self.body_processor:
- self.data = self.body_processor(self.data)
- self.finished.callback(self.data)
+ try:
+ if self.body_processor:
+ self.data = self.body_processor(self.data)
+ self.finished.callback(self.data)
+ except Exception as exc:
+ self.finished.errback(exc)
class Downloader(protocol.Protocol):
def __init__(self, download_path,
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits