[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Start adding support for HTTPS, Cloudfronted test helpers and collectors



commit d17873211da4bd6ec3c0d449ea1d62c2216d1996
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Wed May 11 15:11:13 2016 +0200

    Start adding support for HTTPS, Cloudfronted test helpers and collectors
    
    * Add routines to verify which collectors and test helpers are reachable
    
    * Merciless refactoring of the reporting logic
    
    * Remove dumb logic
---
 ooni/backend_client.py                     | 364 +++++++++++++++++++++++++++++
 ooni/deck.py                               | 164 +++++++++++--
 ooni/director.py                           |  24 +-
 ooni/errors.py                             |  49 +++-
 ooni/nettests/blocking/web_connectivity.py |  36 ++-
 ooni/oonibclient.py                        | 232 ------------------
 ooni/oonicli.py                            |  35 +--
 ooni/report/tool.py                        |   4 +-
 ooni/reporter.py                           | 220 +++++------------
 ooni/tests/mocks.py                        |  12 +-
 ooni/tests/test_deck.py                    |   9 +-
 ooni/tests/test_oonibclient.py             |  47 ++--
 ooni/tests/test_reporter.py                |  32 ++-
 ooni/tests/test_utils.py                   |  34 +--
 ooni/utils/__init__.py                     |  56 ++---
 15 files changed, 729 insertions(+), 589 deletions(-)

diff --git a/ooni/backend_client.py b/ooni/backend_client.py
new file mode 100644
index 0000000..c7de7f0
--- /dev/null
+++ b/ooni/backend_client.py
@@ -0,0 +1,364 @@
+import os
+import json
+
+from urlparse import urljoin, urlparse
+
+from twisted.web.error import Error
+from twisted.web.client import Agent, Headers
+from twisted.internet import defer, reactor
+from twisted.internet.endpoints import TCP4ClientEndpoint
+
+from twisted.python.versions import Version
+from twisted import version as _twisted_version
+_twisted_14_0_2_version = Version('twisted', 14, 0, 2)
+
+from ooni import errors as e
+from ooni.settings import config
+from ooni.utils import log
+from ooni.utils.net import BodyReceiver, StringProducer, Downloader
+from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent
+
+
+class OONIBClient(object):
+    def __init__(self, address=None, settings={}):
+        self.base_headers = {}
+        self.backend_type = settings.get('type', None)
+        self.base_address = settings.get('address', address).encode('ascii')
+
+        if self.backend_type is None:
+            self._guessBackendType()
+        self.backend_type = self.backend_type.encode('ascii')
+
+        if self.backend_type == 'cloudfront':
+            self.base_headers['Host'] = settings['front'].encode('ascii')
+
+        self._setupBaseAddress()
+        self.settings = {
+            'type': self.backend_type,
+            'address': self.base_address,
+            'front': settings.get('front', '').encode('ascii')
+        }
+
+    def _guessBackendType(self):
+        if self.base_address is None:
+            raise e.InvalidAddress
+        if self.base_address.startswith('https://'):
+            self.backend_type = 'https'
+        elif self.base_address.startswith('httpo://'):
+            self.backend_type = 'onion'
+        elif self.base_address.startswith('http://'):
+            self.backend_type = 'http'
+        else:
+            raise e.InvalidAddress
+
+    def _setupBaseAddress(self):
+        parsed_address = urlparse(self.base_address)
+        if self.backend_type == 'onion':
+            if not parsed_address.netloc.endswith(".onion"):
+                log.err("Invalid onion address.")
+                raise e.InvalidAddress(self.base_address)
+            self.base_address = ("http://%s"; % parsed_address.netloc)
+        elif self.backend_type == 'http':
+            self.base_address = ("http://%s"; % parsed_address.netloc)
+        elif self.backend_type in ('https', 'cloudfront'):
+            self.base_address = ("https://%s"; % parsed_address.netloc)
+
+    def isSupported(self):
+        if self.backend_type in ("https", "cloudfront"):
+            if _twisted_version < _twisted_14_0_2_version:
+                log.err("HTTPS and cloudfronted backends require "
+                        "twisted > 14.0.2.")
+                return False
+        elif self.backend_type == "http":
+            if config.advanced.insecure_collector is not True:
+                log.err("Plaintext backends are not supported. To "
+                        "enable at your own risk set "
+                        "advanced->insecure_collector to true")
+                return False
+        elif self.backend_type == "onion":
+            # XXX add an extra check to ensure tor is running
+            if not config.tor_state and config.tor.socks_port is None:
+                return False
+        return True
+
+    def isReachable(self):
+        raise NotImplemented
+
+    def _request(self, method, urn, genReceiver, bodyProducer=None, retries=3):
+        if self.backend_type == 'onion':
+            agent = TrueHeadersSOCKS5Agent(reactor,
+                                           proxyEndpoint=TCP4ClientEndpoint(reactor,
+                                                                            '127.0.0.1',
+                                                                            config.tor.socks_port))
+        else:
+            agent = Agent(reactor)
+
+        attempts = 0
+
+        finished = defer.Deferred()
+
+        def perform_request(attempts):
+            uri = urljoin(self.base_address, urn)
+            d = agent.request(method, uri, bodyProducer=bodyProducer,
+                              headers=Headers(self.base_headers))
+
+            @d.addCallback
+            def callback(response):
+                try:
+                    content_length = int(response.headers.getRawHeaders('content-length')[0])
+                except:
+                    content_length = None
+                response.deliverBody(genReceiver(finished, content_length))
+
+            def errback(err, attempts):
+                # We we will recursively keep trying to perform a request until
+                # we have reached the retry count.
+                if attempts < retries:
+                    log.err("Lookup failed. Retrying.")
+                    attempts += 1
+                    perform_request(attempts)
+                else:
+                    log.err("Failed. Giving up.")
+                    finished.errback(err)
+
+            d.addErrback(errback, attempts)
+
+        perform_request(attempts)
+
+        return finished
+
+    def queryBackend(self, method, urn, query=None, retries=3):
+        bodyProducer = None
+        if query:
+            bodyProducer = StringProducer(json.dumps(query))
+
+        def genReceiver(finished, content_length):
+            def process_response(s):
+                # If empty string then don't parse it.
+                if not s:
+                    return
+                try:
+                    response = json.loads(s)
+                except ValueError:
+                    raise e.get_error(None)
+                if 'error' in response:
+                    log.err("Got this backend error message %s" % response)
+                    raise e.get_error(response['error'])
+                return response
+
+            return BodyReceiver(finished, content_length, process_response)
+
+        return self._request(method, urn, genReceiver, bodyProducer, retries)
+
+    def download(self, urn, download_path):
+
+        def genReceiver(finished, content_length):
+            return Downloader(download_path, finished, content_length)
+
+        return self._request('GET', urn, genReceiver)
+
+class BouncerClient(OONIBClient):
+    def isReachable(self):
+        pass
+
+    @defer.inlineCallbacks
+    def lookupTestCollector(self, net_tests):
+        try:
+            test_collector = yield self.queryBackend('POST', '/bouncer/net-tests',
+                                                     query={'net-tests': net_tests})
+        except Exception as exc:
+            log.exception(exc)
+            raise e.CouldNotFindTestCollector
+
+        defer.returnValue(test_collector)
+
+    @defer.inlineCallbacks
+    def lookupTestHelpers(self, test_helper_names):
+        try:
+            test_helper = yield self.queryBackend('POST', '/bouncer/test-helpers',
+                                                  query={'test-helpers': test_helper_names})
+        except Exception as exc:
+            log.exception(exc)
+            raise e.CouldNotFindTestHelper
+
+        if not test_helper:
+            raise e.CouldNotFindTestHelper
+
+        defer.returnValue(test_helper)
+
+
+class CollectorClient(OONIBClient):
+    def isReachable(self):
+        # XXX maybe in the future we can have a dedicated API endpoint to
+        # test the reachability of the collector.
+        d = self.queryBackend('GET', '/invalidpath')
+
+        @d.addCallback
+        def cb(_):
+            # We should never be getting an acceptable response for a
+            # request to an invalid path.
+            raise e.CollectorUnreachable
+
+        @d.addErrback
+        def err(failure):
+            failure.trap(Error)
+            if failure.value.status == '404':
+                return True
+            raise e.CollectorUnreachable
+
+        return d
+
+    def getInput(self, input_hash):
+        from ooni.deck import InputFile
+
+        input_file = InputFile(input_hash)
+        if input_file.descriptorCached:
+            return defer.succeed(input_file)
+        else:
+            d = self.queryBackend('GET', '/input/' + input_hash)
+
+            @d.addCallback
+            def cb(descriptor):
+                input_file.load(descriptor)
+                input_file.save()
+                return input_file
+
+            @d.addErrback
+            def err(err):
+                log.err("Failed to get descriptor for input %s" % input_hash)
+                log.exception(err)
+
+            return d
+
+    def getInputList(self):
+        return self.queryBackend('GET', '/input')
+
+    def downloadInput(self, input_hash):
+        from ooni.deck import InputFile
+
+        input_file = InputFile(input_hash)
+
+        if input_file.fileCached:
+            return defer.succeed(input_file)
+        else:
+            d = self.download('/input/' + input_hash + '/file', input_file.cached_file)
+
+            @d.addCallback
+            def cb(res):
+                input_file.verify()
+                return input_file
+
+            @d.addErrback
+            def err(err):
+                log.err("Failed to download the input file %s" % input_hash)
+                log.exception(err)
+
+            return d
+
+    def getInputPolicy(self):
+        return self.queryBackend('GET', '/policy/input')
+
+    def getNettestPolicy(self):
+        return self.queryBackend('GET', '/policy/nettest')
+
+    def getDeckList(self):
+        return self.queryBackend('GET', '/deck')
+
+    def getDeck(self, deck_hash):
+        from ooni.deck import Deck
+
+        deck = Deck(deck_hash)
+        if deck.descriptorCached:
+            return defer.succeed(deck)
+        else:
+            d = self.queryBackend('GET', '/deck/' + deck_hash)
+
+            @d.addCallback
+            def cb(descriptor):
+                deck.load(descriptor)
+                deck.save()
+                return deck
+
+            @d.addErrback
+            def err(err):
+                log.err("Failed to get descriptor for deck %s" % deck_hash)
+                log.exception(err)
+
+            return d
+
+    def downloadDeck(self, deck_hash):
+        from ooni.deck import Deck
+
+        deck = Deck(deck_hash)
+        if deck.fileCached:
+            return defer.succeed(deck)
+        else:
+            d = self.download('/deck/' + deck_hash + '/file', deck.cached_file)
+
+            @d.addCallback
+            def cb(res):
+                deck.verify()
+                return deck
+
+            @d.addErrback
+            def err(err):
+                log.err("Failed to download the deck %s" % deck_hash)
+                log.exception(err)
+
+            return d
+
+    def createReport(self, test_details):
+        request = {
+            'software_name': test_details['software_name'],
+            'software_version': test_details['software_version'],
+            'probe_asn': test_details['probe_asn'],
+            'probe_cc': test_details['probe_cc'],
+            'test_name': test_details['test_name'],
+            'test_version': test_details['test_version'],
+            'test_start_time': test_details['test_start_time'],
+            'input_hashes': test_details['input_hashes'],
+            'data_format_version': test_details['data_format_version'],
+            'format': 'json'
+        }
+        # import values from the environment
+        request.update([(k.lower(),v) for (k,v) in os.environ.iteritems()
+                        if k.startswith('PROBE_')])
+
+        return self.queryBackend('POST', '/report', query=request)
+
+    def updateReport(self, report_id, serialization_format, entry_content):
+        request = {
+            'format': serialization_format,
+            'content': entry_content
+        }
+        return self.queryBackend('POST', '/report/%s' % report_id,
+                                 query=request)
+
+
+    def closeReport(self, report_id):
+        return self.queryBackend('POST', '/report/' + report_id + '/close')
+
+class WebConnectivityClient(OONIBClient):
+    def isReachable(self):
+        # XXX maybe in the future we can have a dedicated API endpoint to
+        # test the reachability of the collector.
+        d = self.queryBackend('GET', '/status')
+
+        @d.addCallback
+        def cb(result):
+            if result.get("status", None) is not "ok":
+                raise e.TestHelperUnreachable
+            return True
+
+        @d.addErrback
+        def err(_):
+            raise e.TestHelperUnreachable
+
+        return d
+
+    def control(self, http_request, tcp_connect):
+        request = {
+            'http_request': http_request,
+            'tcp_connect': tcp_connect
+        }
+        self.queryBackend('POST', '/', query=request)
diff --git a/ooni/deck.py b/ooni/deck.py
index 4b2c502..24c7904 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -1,14 +1,12 @@
 # -*- coding: utf-8 -*-
 
-from ooni.oonibclient import OONIBClient
+from ooni.backend_client import CollectorClient, BouncerClient
+from ooni.backend_client import WebConnectivityClient
 from ooni.nettest import NetTestLoader
 from ooni.settings import config
 from ooni.utils import log
 from ooni import errors as e
 
-from twisted import version as _twisted_version
-from twisted.python.versions import Version
-
 from twisted.python.filepath import FilePath
 from twisted.internet import defer
 
@@ -96,7 +94,8 @@ def nettest_to_path(path, allow_arbitrary_paths=False):
 
 class Deck(InputFile):
     # this exists so we can mock it out in unittests
-    _OONIBClient = OONIBClient
+    _BouncerClient = BouncerClient
+    _CollectorClient = CollectorClient
 
     def __init__(self, deck_hash=None,
                  bouncer=None,
@@ -138,7 +137,9 @@ class Deck(InputFile):
                                             annotations=test['options'].get('annotations', {}),
                                             test_file=nettest_path)
             if test['options']['collector']:
-                net_test_loader.collector = test['options']['collector']
+                net_test_loader.collector = CollectorClient(
+                    test['options']['collector']
+                )
             self.insert(net_test_loader)
 
     def insert(self, net_test_loader):
@@ -152,13 +153,6 @@ class Deck(InputFile):
                 raise
             self.requiresTor = True
 
-        if net_test_loader.collector and net_test_loader.collector.startswith('https://'):
-            _twisted_14_0_2_version = Version('twisted', 14, 0, 2)
-            if _twisted_version < _twisted_14_0_2_version:
-                raise e.HTTPCollectorUnsupported
-        elif net_test_loader.collector and net_test_loader.collector.startswith('http://'):
-            if config.advanced.insecure_collector is not True:
-                raise e.InsecureCollector
         self.netTestLoaders.append(net_test_loader)
 
     @defer.inlineCallbacks
@@ -172,9 +166,132 @@ class Deck(InputFile):
             log.msg("Looking up collector and test helpers")
             yield self.lookupCollectorAndTestHelpers()
 
+
+    def sortAddressesByPriority(self, priority_address, alternate_addresses):
+        onion_addresses= []
+        cloudfront_addresses= []
+        https_addresses = []
+        plaintext_addresses = []
+
+        if priority_address.startswith('httpo://'):
+            priority_address = {
+                'address': priority_address,
+                'type': 'onion'
+            }
+        elif priority_address.startswith('https://'):
+            priority_address = {
+                'address': priority_address,
+                'type': 'https'
+            }
+        elif priority_address.startswith('http://'):
+            if config.advanced.insecure_collector is True:
+                priority_address = {
+                    'address': priority_address,
+                    'type': 'http'
+                }
+        else:
+            raise e.InvalidOONIBCollectorAddress
+
+        def filter_by_type(collectors, collector_type):
+            return filter(lambda x: x['type'] == collector_type,
+                          collectors)
+        onion_addresses += filter_by_type(alternate_addresses, 'onion')
+        https_addresses += filter_by_type(alternate_addresses, 'https')
+        cloudfront_addresses += filter_by_type(alternate_addresses,
+                                                'cloudfront')
+
+        if config.advanced.insecure_collector is True:
+            plaintext_addresses += filter_by_type(alternate_addresses, 'http')
+
+        return ([priority_address] +
+                onion_addresses +
+                https_addresses +
+                cloudfront_addresses)
+
+    @defer.inlineCallbacks
+    def getReachableCollector(self, collector_address, collector_alternate):
+        # We prefer onion collector to https collector to cloudfront
+        # collectors to plaintext collectors
+        for collector_settings in self.sortAddressesByPriority(collector_address,
+                                                               collector_alternate):
+            try:
+                collector = self._CollectorClient(settings=collector_settings)
+                if not collector.isSupported():
+                    log.err("Unsupported %s collector %s" % (
+                                collector_settings['type'],
+                                collector_settings['address']))
+                    continue
+                reachable = yield collector.isReachable()
+                if not reachable:
+                    log.err("Unreachable %s collector %s" % (
+                                collector_settings['type'],
+                                collector_settings['address']))
+                    continue
+                defer.returnValue(collector)
+            except e.CollectorUnreachable:
+                log.msg("Could not reach %s collector %s" % (
+                            collector_settings['type'],
+                            collector_settings['address']))
+
+        raise e.NoReachableCollectors
+
+    @defer.inlineCallbacks
+    def getReachableTestHelper(self, test_helper_name, test_helper_address,
+                               test_helper_alternate):
+        # For the moment we look for alternate addresses only of
+        # web_connectivity test helpers.
+        if test_helper_name is 'web_connectivity':
+            for web_connectivity_settings in self.sortAddressesByPriority(
+                    test_helper_address, test_helper_alternate):
+                try:
+                    web_connectivity_test_helper = WebConnectivityClient(web_connectivity_settings)
+                    if not web_connectivity_test_helper.isSupported():
+                        log.err("Unsupported %s web_connectivity test_helper "
+                                "%s" % (
+                                web_connectivity_settings['type'],
+                                web_connectivity_settings['address']
+                        ))
+                        continue
+                    reachable = yield web_connectivity_test_helper.isReachable()
+                    if not reachable:
+                        log.err("Unreachable %s web_connectivity test helper %s" % (
+                            web_connectivity_settings['type'],
+                            web_connectivity_settings['address']
+                        ))
+                        continue
+                    defer.returnValue(web_connectivity_settings)
+                except e.TestHelperUnreachable:
+                    log.err("Unreachable %s web_connectivity test helper %s" % (
+                        web_connectivity_settings['type'],
+                        web_connectivity_settings['address']
+                    ))
+                    continue
+            raise e.NoReachableTestHelpers
+        else:
+            defer.returnValue(test_helper_address.encode('ascii'))
+
+    @defer.inlineCallbacks
+    def getReachableTestHelpersAndCollectors(self, net_tests):
+        for net_test in net_tests:
+            net_test['collector'] = yield self.getReachableCollector(
+                        net_test['collector'],
+                        net_test.get('collector-alternate', [])
+            )
+
+            for test_helper_name, test_helper_address in net_test['test-helpers'].items():
+                 test_helper_alternate = \
+                     net_test.get('test-helpers-alternate', {}).get(test_helper_name, [])
+                 net_test['test-helpers'][test_helper_name] = \
+                            yield self.getReachableTestHelper(
+                                test_helper_name,
+                                test_helper_address,
+                                test_helper_alternate)
+
+        defer.returnValue(net_tests)
+
     @defer.inlineCallbacks
     def lookupCollectorAndTestHelpers(self):
-        oonibclient = self._OONIBClient(self.bouncer)
+        oonibclient = self._BouncerClient(self.bouncer)
 
         required_nettests = []
 
@@ -201,7 +318,14 @@ class Deck(InputFile):
             defer.returnValue(None)
 
         response = yield oonibclient.lookupTestCollector(required_nettests)
-        provided_net_tests = response['net-tests']
+        try:
+            provided_net_tests = yield self.getReachableTestHelpersAndCollectors(response['net-tests'])
+        except e.NoReachableCollectors:
+            log.err("Could not find any reachable collector")
+            raise
+        except e.NoReachableTestHelpers:
+            log.err("Could not find any reachable test helpers")
+            raise
 
         def find_collector_and_test_helpers(test_name, test_version, input_files):
             input_files = [u""+x['hash'] for x in input_files]
@@ -224,12 +348,12 @@ class Deck(InputFile):
                                                 input_files=net_test_loader.inputFiles)
 
             for option, name in net_test_loader.missingTestHelpers:
-                test_helper_address = test_helpers[name].encode('utf-8')
-                net_test_loader.localOptions[option] = test_helper_address
-                net_test_loader.testHelpers[option] = test_helper_address
+                test_helper_address_or_settings = test_helpers[name]
+                net_test_loader.localOptions[option] = test_helper_address_or_settings
+                net_test_loader.testHelpers[option] = test_helper_address_or_settings
 
             if not net_test_loader.collector:
-                net_test_loader.collector = collector.encode('utf-8')
+                net_test_loader.collector = collector
 
     @defer.inlineCallbacks
     def fetchAndVerifyNetTestInput(self, net_test_loader):
@@ -238,7 +362,7 @@ class Deck(InputFile):
         for i in net_test_loader.inputFiles:
             if i['url']:
                 log.debug("Downloading %s" % i['url'])
-                oonibclient = self._OONIBClient(i['address'])
+                oonibclient = self._CollectorClient(i['address'])
 
                 try:
                     input_file = yield oonibclient.downloadInput(i['hash'])
diff --git a/ooni/director.py b/ooni/director.py
index 43ca36c..82d0e85 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -233,7 +233,7 @@ class Director(object):
 
     @defer.inlineCallbacks
     def startNetTest(self, net_test_loader, report_filename,
-                     collector_address=None, no_yamloo=False):
+                     collector_client=None, no_yamloo=False):
         """
         Create the Report for the NetTest and start the report NetTest.
 
@@ -250,7 +250,8 @@ class Director(object):
         if config.privacy.includepcap:
             self.startSniffing(test_details)
         report = Report(test_details, report_filename,
-                        self.reportEntryManager, collector_address,
+                        self.reportEntryManager,
+                        collector_client,
                         no_yamloo)
 
         yield report.open()
@@ -267,7 +268,7 @@ class Director(object):
         finally:
             self.netTestDone(net_test)
 
-    def startSniffing(self, testDetails):
+    def startSniffing(self, test_details):
         """ Start sniffing with Scapy. Exits if required privileges (root) are not
         available.
         """
@@ -276,12 +277,17 @@ class Director(object):
         if config.scapyFactory is None:
             config.scapyFactory = ScapyFactory(config.advanced.interface)
 
-        if not config.reports.pcap:
+        # XXX this is dumb option to have in the ooniprobe.conf. Drop it in
+        # the future.
+        prefix = config.reports.pcap
+        if prefix is None:
             prefix = 'report'
-        else:
-            prefix = config.reports.pcap
-        filename = config.global_options['reportfile'] if 'reportfile' in config.global_options.keys() else None
-        filename_pcap = generate_filename(testDetails, filename=filename, prefix=prefix, extension='pcap')
+
+        filename_pcap = config.global_options.get('pcapfile', None)
+        if filename_pcap is None:
+            filename_pcap = generate_filename(test_details,
+                                              prefix=prefix,
+                                              extension='pcap')
         if len(self.sniffers) > 0:
             pcap_filenames = set(sniffer.pcapwriter.filename for sniffer in self.sniffers.values())
             pcap_filenames.add(filename_pcap)
@@ -289,7 +295,7 @@ class Director(object):
                     ','.join(pcap_filenames))
 
         sniffer = ScapySniffer(filename_pcap)
-        self.sniffers[testDetails['test_name']] = sniffer
+        self.sniffers[test_details['test_name']] = sniffer
         config.scapyFactory.registerProtocol(sniffer)
         log.msg("Starting packet capture to: %s" % filename_pcap)
 
diff --git a/ooni/errors.py b/ooni/errors.py
index 0412b50..e5f26b2 100644
--- a/ooni/errors.py
+++ b/ooni/errors.py
@@ -90,11 +90,14 @@ class UnableToStartTor(DirectorException):
     pass
 
 
-class InvalidOONIBCollectorAddress(Exception):
+class InvalidAddress(Exception):
+    pass
+
+class InvalidOONIBCollectorAddress(InvalidAddress):
     pass
 
 
-class InvalidOONIBBouncerAddress(Exception):
+class InvalidOONIBBouncerAddress(InvalidAddress):
     pass
 
 
@@ -170,6 +173,13 @@ class OONIBInputDescriptorNotFound(OONIBInputError):
     pass
 
 
+class OONIBInvalidInputHash(OONIBError):
+    pass
+
+
+class OONIBInvalidNettestName(OONIBError):
+    pass
+
 class UnableToLoadDeckInput(Exception):
     pass
 
@@ -256,10 +266,14 @@ class ConfigFileIncoherent(Exception):
 def get_error(error_key):
     if error_key == 'test-helpers-key-missing':
         return CouldNotFindTestHelper
-    if error_key == 'input-descriptor-not-found':
+    elif error_key == 'input-descriptor-not-found':
         return OONIBInputDescriptorNotFound
-    if error_key == 'invalid-request':
+    elif error_key == 'invalid-request':
         return OONIBInvalidRequest
+    elif error_key == 'invalid-input-hash':
+        return OONIBInvalidInputHash
+    elif error_key == 'invalid-nettest-name':
+        return OONIBInvalidNettestName
     elif isinstance(error_key, int):
         return Error("%d" % error_key)
     else:
@@ -281,8 +295,33 @@ class ProtocolAlreadyRegistered(Exception):
 class LibraryNotInstalledError(Exception):
     pass
 
+
 class InsecureCollector(Exception):
     pass
 
-class HTTPSCollectorUnsupported(Exception):
+
+class CollectorUnsupported(Exception):
+    pass
+
+class HTTPSCollectorUnsupported(CollectorUnsupported):
+    pass
+
+
+class CollectorUnreachable(Exception):
+    pass
+
+
+class BackendNotSupported(Exception):
+    pass
+
+
+class NoReachableCollectors(Exception):
+    pass
+
+
+class TestHelperUnreachable(Exception):
+    pass
+
+
+class NoReachableTestHelpers(Exception):
     pass
diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index eb05835..da25bdc 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -17,6 +17,8 @@ from twisted.python import usage
 from ooni import geoip
 from ooni.utils import log
 
+from ooni.backend_client import WebConnectivityClient
+
 from ooni.utils.net import StringProducer, BodyReceiver
 from ooni.templates import httpt, dnst
 from ooni.errors import failureToString
@@ -179,6 +181,14 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
                 'headers': {}
             }
         }
+        if isinstance(self.localOptions['backend'], dict):
+            self.web_connectivity_client = WebConnectivityClient(
+                settings=self.localOptions['backend']
+            )
+        else:
+            self.web_connectivity_client = WebConnectivityClient(
+                self.localOptions['backend']
+            )
 
     def experiment_dns_query(self):
         log.msg("* doing DNS query for {}".format(self.hostname))
@@ -214,28 +224,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
     @defer.inlineCallbacks
     def control_request(self, sockets):
-        bodyProducer = StringProducer(json.dumps({
-            'http_request': self.input,
-            'tcp_connect': sockets
-        }))
-        response = yield self.agent.request("POST",
-                                            str(self.localOptions['backend']),
-                                            bodyProducer=bodyProducer)
-        try:
-            content_length = int(response.headers.getRawHeaders('content-length')[0])
-        except Exception:
-            content_length = None
-
-        finished = defer.Deferred()
-        response.deliverBody(BodyReceiver(finished, content_length))
-        body = yield finished
-        try:
-            self.control = json.loads(body)
-            assert 'http_request' in self.control.keys()
-            assert 'tcp_connect' in self.control.keys()
-            assert 'dns' in self.control.keys()
-        except AssertionError, ValueError:
-            raise InvalidControlResponse(body)
+        self.control = yield self.web_connectivity_client.control(
+            http_request=self.input,
+            tcp_connect=sockets
+        )
         self.report['control'] = self.control
 
     def experiment_http_get_request(self):
diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py
deleted file mode 100644
index 336ae4e..0000000
--- a/ooni/oonibclient.py
+++ /dev/null
@@ -1,232 +0,0 @@
-import json
-
-from urlparse import urljoin
-
-from twisted.web.client import Agent
-from twisted.internet import defer, reactor
-from twisted.internet.endpoints import TCP4ClientEndpoint
-
-from ooni import errors as e
-from ooni.settings import config
-from ooni.utils import log
-from ooni.utils.net import BodyReceiver, StringProducer, Downloader
-from ooni.utils.trueheaders import TrueHeadersSOCKS5Agent
-
-
-class OONIBClient(object):
-    retries = 3
-
-    def __init__(self, address):
-        if address.startswith("https://";):
-            log.err("HTTPS bouncers are currently not supported!")
-            raise e.InvalidOONIBBouncerAddress
-        elif address.startswith("http://";):
-            log.msg("Warning using plaintext bouncer!")
-        elif address.startswith("httpo://"):
-            log.debug("Using Tor hidden service bouncer: {}".format(address))
-        else:
-            raise e.InvalidOONIBBouncerAddress
-        self.address = address
-
-    def _request(self, method, urn, genReceiver, bodyProducer=None):
-        address = self.address
-        if self.address.startswith('httpo://'):
-            address = self.address.replace('httpo://', 'http://')
-            agent = TrueHeadersSOCKS5Agent(reactor,
-                                           proxyEndpoint=TCP4ClientEndpoint(reactor, '127.0.0.1',
-                                                                            config.tor.socks_port))
-
-        elif self.address.startswith('https://'):
-            log.err("HTTPS based bouncers are currently not supported.")
-            raise e.InvalidOONIBBouncerAddress
-
-        elif self.address.startswith('http://'):
-            log.msg("Warning using unencrypted bouncer")
-            agent = Agent(reactor)
-
-        attempts = 0
-
-        finished = defer.Deferred()
-
-        def perform_request(attempts):
-            uri = urljoin(address, urn)
-            d = agent.request(method, uri, bodyProducer=bodyProducer)
-
-            @d.addCallback
-            def callback(response):
-                try:
-                    content_length = int(response.headers.getRawHeaders('content-length')[0])
-                except:
-                    content_length = None
-                response.deliverBody(genReceiver(finished, content_length))
-
-            def errback(err, attempts):
-                # We we will recursively keep trying to perform a request until
-                # we have reached the retry count.
-                if attempts < self.retries:
-                    log.err("Lookup failed. Retrying.")
-                    attempts += 1
-                    perform_request(attempts)
-                else:
-                    log.err("Failed. Giving up.")
-                    finished.errback(err)
-
-            d.addErrback(errback, attempts)
-
-        perform_request(attempts)
-
-        return finished
-
-    def queryBackend(self, method, urn, query=None):
-        bodyProducer = None
-        if query:
-            bodyProducer = StringProducer(json.dumps(query))
-
-        def genReceiver(finished, content_length):
-            def process_response(s):
-                # If empty string then don't parse it.
-                if not s:
-                    return
-                try:
-                    response = json.loads(s)
-                except ValueError:
-                    raise e.get_error(None)
-                if 'error' in response:
-                    log.err("Got this backend error message %s" % response)
-                    raise e.get_error(response['error'])
-                return response
-
-            return BodyReceiver(finished, content_length, process_response)
-
-        return self._request(method, urn, genReceiver, bodyProducer)
-
-    def download(self, urn, download_path):
-
-        def genReceiver(finished, content_length):
-            return Downloader(download_path, finished, content_length)
-
-        return self._request('GET', urn, genReceiver)
-
-    def getInput(self, input_hash):
-        from ooni.deck import InputFile
-
-        input_file = InputFile(input_hash)
-        if input_file.descriptorCached:
-            return defer.succeed(input_file)
-        else:
-            d = self.queryBackend('GET', '/input/' + input_hash)
-
-            @d.addCallback
-            def cb(descriptor):
-                input_file.load(descriptor)
-                input_file.save()
-                return input_file
-
-            @d.addErrback
-            def err(err):
-                log.err("Failed to get descriptor for input %s" % input_hash)
-                log.exception(err)
-
-            return d
-
-    def getInputList(self):
-        return self.queryBackend('GET', '/input')
-
-    def downloadInput(self, input_hash):
-        from ooni.deck import InputFile
-
-        input_file = InputFile(input_hash)
-
-        if input_file.fileCached:
-            return defer.succeed(input_file)
-        else:
-            d = self.download('/input/' + input_hash + '/file', input_file.cached_file)
-
-            @d.addCallback
-            def cb(res):
-                input_file.verify()
-                return input_file
-
-            @d.addErrback
-            def err(err):
-                log.err("Failed to download the input file %s" % input_hash)
-                log.exception(err)
-
-            return d
-
-    def getInputPolicy(self):
-        return self.queryBackend('GET', '/policy/input')
-
-    def getNettestPolicy(self):
-        return self.queryBackend('GET', '/policy/nettest')
-
-    def getDeckList(self):
-        return self.queryBackend('GET', '/deck')
-
-    def getDeck(self, deck_hash):
-        from ooni.deck import Deck
-
-        deck = Deck(deck_hash)
-        if deck.descriptorCached:
-            return defer.succeed(deck)
-        else:
-            d = self.queryBackend('GET', '/deck/' + deck_hash)
-
-            @d.addCallback
-            def cb(descriptor):
-                deck.load(descriptor)
-                deck.save()
-                return deck
-
-            @d.addErrback
-            def err(err):
-                log.err("Failed to get descriptor for deck %s" % deck_hash)
-                log.exception(err)
-
-            return d
-
-    def downloadDeck(self, deck_hash):
-        from ooni.deck import Deck
-
-        deck = Deck(deck_hash)
-        if deck.fileCached:
-            return defer.succeed(deck)
-        else:
-            d = self.download('/deck/' + deck_hash + '/file', deck.cached_file)
-
-            @d.addCallback
-            def cb(res):
-                deck.verify()
-                return deck
-
-            @d.addErrback
-            def err(err):
-                log.err("Failed to download the deck %s" % deck_hash)
-                log.exception(err)
-
-            return d
-
-    @defer.inlineCallbacks
-    def lookupTestCollector(self, net_tests):
-        try:
-            test_collector = yield self.queryBackend('POST', '/bouncer/net-tests',
-                                                     query={'net-tests': net_tests})
-        except Exception as exc:
-            log.exception(exc)
-            raise e.CouldNotFindTestCollector
-
-        defer.returnValue(test_collector)
-
-    @defer.inlineCallbacks
-    def lookupTestHelpers(self, test_helper_names):
-        try:
-            test_helper = yield self.queryBackend('POST', '/bouncer/test-helpers',
-                                                  query={'test-helpers': test_helper_names})
-        except Exception as exc:
-            log.exception(exc)
-            raise e.CouldNotFindTestHelper
-
-        if not test_helper:
-            raise e.CouldNotFindTestHelper
-
-        defer.returnValue(test_helper)
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 281ff46..ca8774f 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -13,6 +13,7 @@ from twisted.internet import defer
 from ooni import errors, __version__
 from ooni.settings import config
 from ooni.utils import log
+from backend_client import CollectorClient
 
 class LifetimeExceeded(Exception): pass
 
@@ -222,18 +223,14 @@ def setupAnnotations(global_options):
     global_options["annotations"] = annotations
     return annotations
 
-def setupCollector(global_options, collector_address):
+def setupCollector(global_options, collector_client):
     if global_options['collector']:
-        collector_address = global_options['collector']
-    elif 'collector' in config.reports \
-            and config.reports['collector']:
-        collector_address = config.reports['collector']
-
-    if collector_address.startswith('httpo:') \
-            and (not (config.tor_state or config.tor.socks_port)):
-        raise errors.TorNotRunning
-    return collector_address
-
+        collector_client = CollectorClient(global_options['collector'])
+    elif config.reports.get('collector', None) is not None:
+        collector_client = CollectorClient(config.reports['collector'])
+    if not collector_client.isSupported():
+        raise errors.CollectorUnsupported
+    return collector_client
 
 def createDeck(global_options, url=None):
     from ooni.nettest import NetTestLoader
@@ -264,7 +261,8 @@ def createDeck(global_options, url=None):
                                             test_file=test_file,
                                             annotations=global_options['annotations'])
             if global_options['collector']:
-                net_test_loader.collector = global_options['collector']
+                net_test_loader.collector = \
+                    CollectorClient(global_options['collector'])
             deck.insert(net_test_loader)
     except errors.MissingRequiredOption as option_name:
         log.err('Missing required option: "%s"' % option_name)
@@ -309,7 +307,10 @@ def runTestWithDirector(director, global_options, url=None, start_tor=True):
             return deck.setup()
         except errors.UnableToLoadDeckInput as error:
             return defer.failure.Failure(error)
-
+        except errors.NoReachableTestHelpers as error:
+            return defer.failure.Failure(error)
+        except errors.NoReachableCollectors as error:
+            return defer.failure.Failure(error)
 
     # Wait until director has started up (including bootstrapping Tor)
     # before adding tests
@@ -324,14 +325,14 @@ def runTestWithDirector(director, global_options, url=None, start_tor=True):
             # If a collector is not specified in the deck, or the
             # deck is a singleton, the default collector set in
             # ooniprobe.conf will be used
-            collector_address = None
+            collector_client = None
             if not global_options['no-collector']:
-                collector_address = setupCollector(global_options,
-                                                   net_test_loader.collector)
+                collector_client = setupCollector(global_options,
+                                                  net_test_loader.collector)
 
             yield director.startNetTest(net_test_loader,
                                         global_options['reportfile'],
-                                        collector_address,
+                                        collector_client,
                                         global_options['no-yamloo'])
 
     d.addCallback(setup_nettest)
diff --git a/ooni/report/tool.py b/ooni/report/tool.py
index 5c7bcb2..fd504a6 100644
--- a/ooni/report/tool.py
+++ b/ooni/report/tool.py
@@ -9,7 +9,7 @@ from ooni.reporter import OONIBReporter, OONIBReportLog
 from ooni.utils import log
 from ooni.report import parser
 from ooni.settings import config
-from ooni.oonibclient import OONIBClient
+from ooni.backend_client import BouncerClient
 
 
 @defer.inlineCallbacks
@@ -23,7 +23,7 @@ def upload(report_file, collector=None, bouncer=None):
 
     report = parser.ReportLoader(report_file)
     if bouncer and not collector:
-        oonib_client = OONIBClient(bouncer)
+        oonib_client = BouncerClient(bouncer)
         net_tests = [{
             'test-helpers': [],
             'input-hashes': report.header['input_hashes'],
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 6103c1e..70c2f56 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -1,8 +1,6 @@
 import uuid
 import yaml
-import json
 import os
-import re
 
 from copy import deepcopy
 
@@ -15,12 +13,8 @@ from yaml.serializer import Serializer
 from yaml.resolver import Resolver
 
 from twisted.python.util import untilConcludes
-from twisted.internet import defer, reactor
-from twisted.web.client import Agent
+from twisted.internet import defer
 from twisted.internet.error import ConnectionRefusedError
-from twisted.internet.endpoints import TCP4ClientEndpoint
-
-from txsocksx.http import SOCKS5Agent
 
 from ooni.utils import log
 from ooni.tasks import Measurement
@@ -35,8 +29,7 @@ except ImportError:
 from ooni import errors
 
 from ooni import otime
-from ooni.utils import pushFilenameStack, generate_filename
-from ooni.utils.net import BodyReceiver, StringProducer
+from ooni.utils import generate_filename
 
 from ooni.settings import config
 
@@ -147,6 +140,7 @@ class OReporter(object):
     def finish(self):
         pass
 
+
 class YAMLReporter(OReporter):
 
     """
@@ -157,24 +151,8 @@ class YAMLReporter(OReporter):
 
     """
 
-    def __init__(self, test_details, report_destination='.', report_filename=None):
-        self.reportDestination = report_destination
-
-        if not os.path.isdir(report_destination):
-            raise errors.InvalidDestination
-
-        report_filename = generate_filename(test_details,
-                                            filename=report_filename,
-                                            prefix='report',
-                                            extension='yamloo')
-
-        report_path = os.path.join(self.reportDestination, report_filename)
-
-        if os.path.exists(report_path):
-            log.msg("Report already exists with filename %s" % report_path)
-            pushFilenameStack(report_path)
-
-        self.report_path = os.path.abspath(report_path)
+    def __init__(self, test_details, report_filename):
+        self.report_path = report_filename
         OReporter.__init__(self, test_details)
 
     def _writeln(self, line):
@@ -229,42 +207,15 @@ class YAMLReporter(OReporter):
         self._stream.close()
 
 
-def collector_supported(collector_address):
-    if collector_address.startswith('httpo') \
-            and (not (config.tor_state or config.tor.socks_port)):
-        return False
-    return True
-
-
 class OONIBReporter(OReporter):
 
-    def __init__(self, test_details, collector_address):
-        self.collectorAddress = collector_address
-        self.validateCollectorAddress()
+    def __init__(self, test_details, collector_client):
+        self.collector_client = collector_client
 
         self.reportId = None
         self.supportedFormats = ["yaml"]
-
-        if self.collectorAddress.startswith('https://'):
-            # not sure if there's something else it needs.  Seems to work.
-            # Very difficult to get it to work with self-signed certs.
-            self.agent = Agent(reactor)
-
-        elif self.collectorAddress.startswith('http://'):
-            log.msg("Warning using unencrypted collector")
-            self.agent = Agent(reactor)
-
         OReporter.__init__(self, test_details)
 
-    def validateCollectorAddress(self):
-        """
-        Will raise :class:ooni.errors.InvalidOONIBCollectorAddress an exception
-        if the oonib reporter is not valid.
-        """
-        regexp = '^(http|https|httpo):\/\/[a-zA-Z0-9\-\.]+(:\d+)?$'
-        if not re.match(regexp, self.collectorAddress):
-            raise errors.InvalidOONIBCollectorAddress
-
     def serializeEntry(self, entry, serialisation_format="yaml"):
         if serialisation_format == "json":
             if isinstance(entry, Measurement):
@@ -303,29 +254,17 @@ class OONIBReporter(OReporter):
 
     @defer.inlineCallbacks
     def writeReportEntry(self, entry):
-        log.debug("Writing report with OONIB reporter")
-
-        url = self.collectorAddress + '/report/' + self.reportId
-
         if "json" in self.supportedFormats:
-            serialisation_format = 'json'
+            serialization_format = 'json'
         else:
-            serialisation_format = 'yaml'
-
-        request = {
-            'format': serialisation_format,
-            'content': self.serializeEntry(entry, serialisation_format)
-        }
-
-        log.debug("Updating report with id %s (%s)" % (self.reportId, url))
-        request_json = json.dumps(request)
-        log.debug("Sending %s" % request_json)
-
-        bodyProducer = StringProducer(request_json)
+            serialization_format = 'yaml'
 
+        log.debug("Updating report with id %s" % (self.reportId))
+        entry_content = self.serializeEntry(entry, serialization_format)
         try:
-            yield self.agent.request("POST", str(url),
-                                     bodyProducer=bodyProducer)
+            yield self.collector_client.updateReport(self.reportId,
+                                                     serialization_format,
+                                                     entry_content)
         except Exception as exc:
             log.err("Error in writing report entry")
             log.exception(exc)
@@ -336,100 +275,43 @@ class OONIBReporter(OReporter):
         """
         Creates a report on the oonib collector.
         """
-        # XXX we should probably be setting this inside of the constructor,
-        # however config.tor.socks_port is not set until Tor is started and the
-        # reporter is instantiated before Tor is started. We probably want to
-        # do this with some deferred kung foo or instantiate the reporter after
-        # tor is started.
-
-
-        if self.collectorAddress.startswith('httpo://'):
-            self.collectorAddress = \
-                self.collectorAddress.replace('httpo://', 'http://')
-            proxyEndpoint = TCP4ClientEndpoint(reactor, '127.0.0.1',
-                                               config.tor.socks_port)
-            self.agent = SOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)
-
-        url = self.collectorAddress + '/report'
-
-        request = {
-            'software_name': self.testDetails['software_name'],
-            'software_version': self.testDetails['software_version'],
-            'probe_asn': self.testDetails['probe_asn'],
-            'probe_cc': self.testDetails['probe_cc'],
-            'test_name': self.testDetails['test_name'],
-            'test_version': self.testDetails['test_version'],
-            'test_start_time': self.testDetails['test_start_time'],
-            'input_hashes': self.testDetails['input_hashes'],
-            'data_format_version': self.testDetails['data_format_version'],
-            'format': 'json'
-        }
-        # import values from the environment
-        request.update([(k.lower(),v) for (k,v) in os.environ.iteritems()
-                        if k.startswith('PROBE_')])
-
-        log.msg("Reporting %s" % url)
-        request_json = json.dumps(request)
-        log.debug("Sending %s" % request_json)
-
-        bodyProducer = StringProducer(request_json)
-
         log.msg("Creating report with OONIB Reporter. Please be patient.")
         log.msg("This may take up to 1-2 minutes...")
 
         try:
-            response = yield self.agent.request("POST", url,
-                                                bodyProducer=bodyProducer)
-
+            response = yield self.collector_client.createReport(
+                                            self.testDetails
+            )
         except ConnectionRefusedError:
             log.err("Connection to reporting backend failed "
                     "(ConnectionRefusedError)")
             raise errors.OONIBReportCreationError
-
         except errors.HostUnreachable:
             log.err("Host is not reachable (HostUnreachable error")
             raise errors.OONIBReportCreationError
-
-        except Exception, e:
-            log.err("Failed to connect to reporter backend")
-            log.exception(e)
-            raise errors.OONIBReportCreationError
-
-        # This is a little trix to allow us to unspool the response. We create
-        # a deferred and call yield on it.
-        response_body = defer.Deferred()
-        response.deliverBody(BodyReceiver(response_body))
-
-        backend_response = yield response_body
-
-        try:
-            parsed_response = json.loads(backend_response)
-        except Exception, e:
-            log.err("Failed to parse collector response %s" % backend_response)
-            log.exception(e)
-            raise errors.OONIBReportCreationError
-
-        if response.code == 406:
-            # XXX make this more strict
+        except (errors.OONIBInvalidInputHash,
+                errors.OONIBInvalidNettestName):
             log.err("The specified input or nettests cannot be submitted to "
                     "this collector.")
             log.msg("Try running a different test or try reporting to a "
                     "different collector.")
             raise errors.OONIBReportCreationError
+        except Exception, e:
+            log.err("Failed to connect to reporter backend")
+            log.exception(e)
+            raise errors.OONIBReportCreationError
 
-        self.reportId = parsed_response['report_id']
-        self.backendVersion = parsed_response['backend_version']
+        self.reportId = response['report_id'].encode('ascii')
+        self.backendVersion = response['backend_version']
 
-        self.supportedFormats = parsed_response.get('supported_formats', ["yaml"])
+        self.supportedFormats = response.get('supported_formats', ["yaml"])
 
-        log.debug("Created report with id %s" % parsed_response['report_id'])
-        defer.returnValue(parsed_response['report_id'])
+        log.debug("Created report with id %s" % response['report_id'])
+        defer.returnValue(response['report_id'])
 
     def finish(self):
-        url = self.collectorAddress + '/report/' + self.reportId + '/close'
-        log.debug("Closing the report %s" % url)
-        return self.agent.request("POST", str(url))
-
+        log.debug("Closing report with id %s" % self.reportId)
+        return self.collector_client.closeReport(self.reportId)
 
 class OONIBReportLog(object):
 
@@ -532,33 +414,34 @@ class OONIBReportLog(object):
     def not_created(self, report_file):
         return self.run(self._not_created, report_file)
 
-    def _created(self, report_file, collector_address, report_id):
+    def _created(self, report_file, collector_settings, report_id):
         with self.edit_log() as report:
+            assert report_file is not None
             report[report_file] = {
                 'pid': os.getpid(),
                 'created_at': datetime.now(),
                 'status': 'created',
-                'collector': collector_address,
+                'collector': collector_settings,
                 'report_id': report_id
             }
         return report_id
 
-    def created(self, report_file, collector_address, report_id):
+    def created(self, report_file, collector_settings, report_id):
         return self.run(self._created, report_file,
-                        collector_address, report_id)
+                        collector_settings, report_id)
 
-    def _creation_failed(self, report_file, collector_address):
+    def _creation_failed(self, report_file, collector_settings):
         with self.edit_log() as report:
             report[report_file] = {
                 'pid': os.getpid(),
                 'created_at': datetime.now(),
                 'status': 'creation-failed',
-                'collector': collector_address
+                'collector': collector_settings
             }
 
-    def creation_failed(self, report_file, collector_address):
+    def creation_failed(self, report_file, collector_settings):
         return self.run(self._creation_failed, report_file,
-                        collector_address)
+                        collector_settings)
 
     def _incomplete(self, report_file):
         with self.edit_log() as report:
@@ -583,7 +466,7 @@ class Report(object):
     reportId = None
 
     def __init__(self, test_details, report_filename,
-                 reportEntryManager, collector_address=None,
+                 reportEntryManager, collector_client=None,
                  no_yamloo=False):
         """
         This is an abstraction layer on top of all the configured reporters.
@@ -608,7 +491,9 @@ class Report(object):
                 If we should disable reporting to disk.
         """
         self.test_details = test_details
-        self.collector_address = collector_address
+        self.collector_client = collector_client
+        if report_filename is None:
+            report_filename = self.generateReportFilename()
         self.report_filename = report_filename
 
         self.report_log = OONIBReportLog()
@@ -620,18 +505,25 @@ class Report(object):
         self.done = defer.Deferred()
         self.reportEntryManager = reportEntryManager
 
+    def generateReportFilename(self):
+        report_filename = generate_filename(self.test_details,
+                                            prefix='report',
+                                            extension='yamloo')
+        report_path = os.path.join('.', report_filename)
+        return os.path.abspath(report_path)
+
     def open_oonib_reporter(self):
         def creation_failed(failure):
             self.oonib_reporter = None
             return self.report_log.creation_failed(self.report_filename,
-                                                   self.collector_address)
+                                                   self.collector_client.settings)
 
         def created(report_id):
             if not self.oonib_reporter:
                 return
             self.test_details['report_id'] = report_id
             return self.report_log.created(self.report_filename,
-                                           self.collector_address,
+                                           self.collector_client.settings,
                                            report_id)
 
         d = self.oonib_reporter.createReport()
@@ -645,15 +537,14 @@ class Report(object):
         This will create all the reports that need to be created and fires the
         created callback of the reporter whose report got created.
         """
-        if self.collector_address:
+        if self.collector_client:
             self.oonib_reporter = OONIBReporter(self.test_details,
-                                                self.collector_address)
+                                                self.collector_client)
             self.test_details['report_id'] = yield self.open_oonib_reporter()
 
         if not self.no_yamloo:
             self.yaml_reporter = YAMLReporter(self.test_details,
-                                              report_filename=self.report_filename)
-            self.report_filename = self.yaml_reporter.report_path
+                                              self.report_filename)
             if not self.oonib_reporter:
                 yield self.report_log.not_created(self.report_filename)
             yield defer.maybeDeferred(self.yaml_reporter.createReport)
@@ -724,6 +615,7 @@ class Report(object):
             return self.report_log.closed(self.report_filename)
 
         def oonib_report_failed(result):
+            log.exception(result)
             log.err("Failed to close oonib report.")
 
         def all_reports_closed(_):
diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py
index 19b4692..f3f852f 100644
--- a/ooni/tests/mocks.py
+++ b/ooni/tests/mocks.py
@@ -3,7 +3,7 @@ from twisted.internet import defer
 
 from ooni.tasks import BaseTask, TaskWithTimeout
 from ooni.managers import TaskManager
-
+from ooni.backend_client import CollectorClient
 
 class MockMeasurementFailOnce(BaseTask):
     def run(self):
@@ -189,7 +189,7 @@ class MockTaskManager(TaskManager):
         self.successes.append((result, task))
 
 
-class MockOONIBClient(object):
+class MockBouncerClient(object):
     def __init__(self, *args, **kw):
         pass
 
@@ -225,3 +225,11 @@ class MockOONIBClient(object):
                 'test-helpers': test_helpers
             })
         return defer.succeed(ret)
+
+
+class MockCollectorClient(CollectorClient):
+    def isSupported(self):
+        return True
+
+    def isReachable(self):
+        return defer.succeed(True)
diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py
index 3e2a322..8d415f0 100644
--- a/ooni/tests/test_deck.py
+++ b/ooni/tests/test_deck.py
@@ -5,7 +5,7 @@ from twisted.trial import unittest
 
 from hashlib import sha256
 from ooni.deck import InputFile, Deck
-from ooni.tests.mocks import MockOONIBClient
+from ooni.tests.mocks import MockBouncerClient, MockCollectorClient
 
 net_test_string = """
 from twisted.python import usage
@@ -151,15 +151,16 @@ class TestDeck(BaseTestCase):
     def test_lookup_test_helpers_and_collector(self):
         deck = Deck(bouncer="httpo://foo.onion",
                     decks_directory=".")
-        deck._OONIBClient = MockOONIBClient
+        deck._BouncerClient = MockBouncerClient
+        deck._CollectorClient = MockCollectorClient
         deck.loadDeck(self.deck_file)
 
         self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
 
         yield deck.lookupCollectorAndTestHelpers()
 
-        self.assertEqual(deck.netTestLoaders[0].collector,
-                         'httpo://thirteenchars1234.onion')
+        self.assertEqual(deck.netTestLoaders[0].collector.settings['address'],
+                         'http://thirteenchars1234.onion')
 
         self.assertEqual(deck.netTestLoaders[0].localOptions['backend'],
                          '127.0.0.1')
diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py
index d064d0a..a14a881 100644
--- a/ooni/tests/test_oonibclient.py
+++ b/ooni/tests/test_oonibclient.py
@@ -7,7 +7,7 @@ from twisted.web import error
 
 from ooni import errors as e
 from ooni.settings import config
-from ooni.oonibclient import OONIBClient
+from ooni.backend_client import CollectorClient, BouncerClient
 from ooni.tests.bases import ConfigTestCase
 
 input_id = '37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1'
@@ -34,79 +34,81 @@ class TestOONIBClient(ConfigTestCase):
             os.mkdir(os.path.join(data_dir, 'decks'))
         except Exception:
             self.skipTest("OONIB must be listening on port 8888 to run this test (tor_hidden_service: false)")
-        self.oonibclient = OONIBClient('http://' + host + ':' + str(port))
+        self.collector_client = CollectorClient('http://' + host + ':' + str(port))
 
     @defer.inlineCallbacks
     def test_query(self):
-        res = yield self.oonibclient.queryBackend('GET', '/policy/input')
+        res = yield self.collector_client.queryBackend('GET', '/policy/input')
         self.assertTrue(isinstance(res, list))
 
     @defer.inlineCallbacks
     def test_get_input_list(self):
-        input_list = yield self.oonibclient.getInputList()
+        input_list = yield self.collector_client.getInputList()
         self.assertTrue(isinstance(input_list, list))
 
     @defer.inlineCallbacks
     def test_get_input_descriptor(self):
-        input_descriptor = yield self.oonibclient.getInput(input_id)
+        input_descriptor = yield self.collector_client.getInput(input_id)
         for key in ['name', 'description',
                     'version', 'author', 'date', 'id']:
             self.assertTrue(hasattr(input_descriptor, key))
 
     @defer.inlineCallbacks
     def test_download_input(self):
-        yield self.oonibclient.downloadInput(input_id)
+        yield self.collector_client.downloadInput(input_id)
 
     @defer.inlineCallbacks
     def test_get_deck_list(self):
-        deck_list = yield self.oonibclient.getDeckList()
+        deck_list = yield self.collector_client.getDeckList()
         self.assertTrue(isinstance(deck_list, list))
 
     @defer.inlineCallbacks
     def test_get_deck_descriptor(self):
-        deck_descriptor = yield self.oonibclient.getDeck(deck_id)
+        deck_descriptor = yield self.collector_client.getDeck(deck_id)
         for key in ['name', 'description',
                     'version', 'author', 'date', 'id']:
             self.assertTrue(hasattr(deck_descriptor, key))
 
     @defer.inlineCallbacks
     def test_download_deck(self):
-        yield self.oonibclient.downloadDeck(deck_id)
+        yield self.collector_client.downloadDeck(deck_id)
 
     def test_lookup_invalid_helpers(self):
-        self.oonibclient.address = 'http://127.0.0.1:8888'
+        bouncer_client = BouncerClient('http://127.0.0.1:8888')
         return self.failUnlessFailure(
-            self.oonibclient.lookupTestHelpers([
+            bouncer_client.lookupTestHelpers([
                 'sdadsadsa', 'dns'
             ]), e.CouldNotFindTestHelper)
 
     @defer.inlineCallbacks
     def test_lookup_no_test_helpers(self):
-        self.oonibclient.address = 'http://127.0.0.1:8888'
+        bouncer_client = BouncerClient('http://127.0.0.1:8888')
         required_helpers = []
-        helpers = yield self.oonibclient.lookupTestHelpers(required_helpers)
+        helpers = yield bouncer_client.lookupTestHelpers(required_helpers)
         self.assertTrue('default' in helpers.keys())
 
     @defer.inlineCallbacks
     def test_lookup_test_helpers(self):
-        self.oonibclient.address = 'http://127.0.0.1:8888'
+        bouncer_client = BouncerClient('http://127.0.0.1:8888')
         required_helpers = [u'http-return-json-headers', u'dns']
-        helpers = yield self.oonibclient.lookupTestHelpers(required_helpers)
+        helpers = yield bouncer_client.lookupTestHelpers(required_helpers)
         self.assertEqual(set(helpers.keys()), set(required_helpers + [u'default']))
         self.assertTrue(helpers['http-return-json-headers']['address'].startswith('http'))
         self.assertTrue(int(helpers['dns']['address'].split('.')[0]))
 
     @defer.inlineCallbacks
     def test_input_descriptor_not_found(self):
-        yield self.assertFailure(self.oonibclient.queryBackend('GET', '/input/' + 'a'*64), e.OONIBInputDescriptorNotFound)
+        yield self.assertFailure(self.collector_client.queryBackend('GET',
+                                                             '/input/' + 'a'*64), e.OONIBInputDescriptorNotFound)
 
     @defer.inlineCallbacks
     def test_http_errors(self):
-        yield self.assertFailure(self.oonibclient.queryBackend('PUT', '/policy/input'), error.Error)
+        yield self.assertFailure(self.collector_client.queryBackend('PUT',
+                                                     '/policy/input'), error.Error)
 
     @defer.inlineCallbacks
     def test_create_report(self):
-        res = yield self.oonibclient.queryBackend('POST', '/report', {
+        res = yield self.collector_client.queryBackend('POST', '/report', {
             'software_name': 'spam',
             'software_version': '2.0',
             'probe_asn': 'AS0',
@@ -119,7 +121,7 @@ class TestOONIBClient(ConfigTestCase):
 
     @defer.inlineCallbacks
     def test_report_lifecycle(self):
-        res = yield self.oonibclient.queryBackend('POST', '/report', {
+        res = yield self.collector_client.queryBackend('POST', '/report', {
             'software_name': 'spam',
             'software_version': '2.0',
             'probe_asn': 'AS0',
@@ -130,12 +132,13 @@ class TestOONIBClient(ConfigTestCase):
         })
         report_id = str(res['report_id'])
 
-        res = yield self.oonibclient.queryBackend('POST', '/report/' + report_id, {
+        res = yield self.collector_client.queryBackend('POST', '/report/' + report_id, {
             'content': '---\nspam: ham\n...\n'
         })
 
-        res = yield self.oonibclient.queryBackend('POST', '/report/' + report_id, {
+        res = yield self.collector_client.queryBackend('POST', '/report/' + report_id, {
             'content': '---\nspam: ham\n...\n'
         })
 
-        res = yield self.oonibclient.queryBackend('POST', '/report/' + report_id + '/close')
+        res = yield self.collector_client.queryBackend('POST', '/report/' + report_id +
+                                        '/close')
diff --git a/ooni/tests/test_reporter.py b/ooni/tests/test_reporter.py
index b4e7592..8f32733 100644
--- a/ooni/tests/test_reporter.py
+++ b/ooni/tests/test_reporter.py
@@ -8,6 +8,7 @@ from twisted.internet import defer
 from twisted.trial import unittest
 
 from ooni import errors as e
+from ooni.tests.mocks import MockCollectorClient
 from ooni.reporter import YAMLReporter, OONIBReporter, OONIBReportLog
 
 
@@ -58,7 +59,7 @@ class TestYAMLReporter(unittest.TestCase):
             os.remove(self.filename)
 
     def test_write_report(self):
-        y_reporter = YAMLReporter(test_details)
+        y_reporter = YAMLReporter(test_details, 'dummy-report.yaml')
         y_reporter.createReport()
         with open(y_reporter.report_path) as f:
             self.filename = y_reporter.report_path
@@ -72,33 +73,30 @@ class TestOONIBReporter(unittest.TestCase):
 
     def setUp(self):
         self.mock_response = {}
-        self.collector_address = 'http://example.com'
 
-        self.oonib_reporter = OONIBReporter(
-            test_details,
-            self.collector_address)
-        self.oonib_reporter.agent = MagicMock()
-        self.mock_agent_response = MagicMock()
+        def mockRequest(method, urn, genReceiver, *args, **kw):
+            receiver = genReceiver(None, None)
+            return defer.maybeDeferred(receiver.body_processor,
+                                       json.dumps(self.mock_response))
 
-        def deliverBody(body_receiver):
-            body_receiver.dataReceived(json.dumps(self.mock_response))
-            body_receiver.connectionLost(None)
+        mock_collector_client = MockCollectorClient('http://example.com')
+        mock_collector_client._request = mockRequest
 
-        self.mock_agent_response.deliverBody = deliverBody
-        self.oonib_reporter.agent.request.return_value = defer.succeed(
-            self.mock_agent_response)
+        self.oonib_reporter = OONIBReporter(
+            test_details,
+            mock_collector_client
+        )
 
     @defer.inlineCallbacks
     def test_create_report(self):
         self.mock_response = oonib_new_report_message
         yield self.oonib_reporter.createReport()
-        assert self.oonib_reporter.reportId == oonib_new_report_message[
-            'report_id']
+        self.assertEqual(self.oonib_reporter.reportId,
+                         oonib_new_report_message['report_id'])
 
     @defer.inlineCallbacks
     def test_create_report_failure(self):
         self.mock_response = oonib_generic_error_message
-        self.mock_agent_response.code = 406
         yield self.assertFailure(self.oonib_reporter.createReport(),
                                  e.OONIBReportCreationError)
 
@@ -108,7 +106,6 @@ class TestOONIBReporter(unittest.TestCase):
         yield self.oonib_reporter.createReport()
         req = {'content': 'something'}
         yield self.oonib_reporter.writeReportEntry(req)
-        assert self.oonib_reporter.agent.request.called
 
     @defer.inlineCallbacks
     def test_write_report_entry_in_yaml(self):
@@ -116,7 +113,6 @@ class TestOONIBReporter(unittest.TestCase):
         yield self.oonib_reporter.createReport()
         req = {'content': 'something'}
         yield self.oonib_reporter.writeReportEntry(req)
-        assert self.oonib_reporter.agent.request.called
 
 class TestOONIBReportLog(unittest.TestCase):
 
diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py
index 855eb19..bbaa26b 100644
--- a/ooni/tests/test_utils.py
+++ b/ooni/tests/test_utils.py
@@ -1,7 +1,7 @@
 import os
 from twisted.trial import unittest
 
-from ooni.utils import pushFilenameStack, log, generate_filename, net
+from ooni.utils import log, generate_filename, net
 
 
 class TestUtils(unittest.TestCase):
@@ -15,26 +15,6 @@ class TestUtils(unittest.TestCase):
         self.basename = 'filename'
         self.filename = 'filename.txe'
 
-    def test_pushFilenameStack(self):
-        basefilename = os.path.join(os.getcwd(), 'dummyfile')
-        f = open(basefilename, "w+")
-        f.write("0\n")
-        f.close()
-        for i in xrange(1, 20):
-            f = open("%s.%d" % (basefilename, i), "w+")
-            f.write("%s\n" % i)
-            f.close()
-
-        pushFilenameStack(basefilename)
-        for i in xrange(1, 20):
-            f = open("%s.%d" % (basefilename, i))
-            c = f.readlines()[0].strip()
-            self.assertEqual(str(i-1), str(c))
-            f.close()
-
-        for i in xrange(1, 21):
-            os.remove("%s.%d" % (basefilename, i))
-
     def test_log_encode(self):
         logmsgs = (
             (r"spam\x07\x08", "spam\a\b"),
@@ -60,18 +40,6 @@ class TestUtils(unittest.TestCase):
         filename = generate_filename(self.test_details, prefix=self.prefix, extension=self.extension)
         self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z.ext')
 
-    def test_generate_filename_with_filename(self):
-        filename = generate_filename(self.test_details, filename=self.filename)
-        self.assertEqual(filename, 'filename.txe')
-
-    def test_generate_filename_with_extension_and_filename(self):
-        filename = generate_filename(self.test_details, extension=self.extension, filename=self.filename)
-        self.assertEqual(filename, 'filename.ext')
-
-    def test_generate_filename_with_extension_and_basename(self):
-        filename = generate_filename(self.test_details, extension=self.extension, filename=self.basename)
-        self.assertEqual(filename, 'filename.ext')
-
     def test_get_addresses(self):
         addresses = net.getAddresses()
         assert isinstance(addresses, list)
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 78a1c7b..22af062 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -91,51 +91,29 @@ def randomStr(length, num=True):
         chars += string.digits
     return ''.join(random.choice(chars) for x in range(length))
 
-
-def pushFilenameStack(filename):
-    """
-    Takes as input a target filename and checks to see if a file by such name
-    already exists. If it does exist then it will attempt to rename it to .1,
-    if .1 exists it will rename .1 to .2 if .2 exists then it will rename it to
-    .3, etc.
-    This is similar to pushing into a LIFO stack.
-
-    Args:
-        filename (str): the path to filename that you wish to create.
-    """
-    stack = glob.glob(filename + ".*")
-    stack.sort(key=lambda x: int(x.split('.')[-1]))
-    for f in reversed(stack):
-        c_idx = f.split(".")[-1]
-        c_filename = '.'.join(f.split(".")[:-1])
-        new_idx = int(c_idx) + 1
-        new_filename = "%s.%s" % (c_filename, new_idx)
-        os.rename(f, new_filename)
-    os.rename(filename, filename + ".1")
-
-
-def generate_filename(testDetails, prefix=None, extension=None, filename=None):
+def generate_filename(test_details, prefix=None, extension=None):
     """
     Returns a filename for every test execution.
 
     It's used to assure that all files of a certain test have a common basename but different
     extension.
     """
-    if filename is None:
-        test_name, start_time = testDetails['test_name'], testDetails['test_start_time']
-        start_time = datetime.strptime(start_time, "%Y-%m-%d %H:%M:%S").strftime("%Y-%m-%dT%H%M%SZ")
-        suffix = "%s-%s" % (test_name, start_time)
-        basename = '%s-%s' % (prefix, suffix) if prefix is not None else suffix
-        final_filename = '%s.%s' % (basename, extension) if extension is not None else basename
-    else:
-        if extension is not None:
-            basename = filename.split('.')[0] if '.' in filename else filename
-            final_filename = '%s.%s' % (basename, extension)
-        else:
-            final_filename = filename
-
-    return final_filename
-
+    LONG_DATE = "%Y-%m-%d %H:%M:%S"
+    SHORT_DATE = "%Y-%m-%dT%H%M%SZ"
+
+    kwargs = {}
+    filename_format = ""
+    if prefix is not None:
+        kwargs["prefix"] = prefix
+        filename_format += "{prefix}-"
+    filename_format += "{test_name}-{timestamp}"
+    if extension is not None:
+        kwargs["extension"] = extension
+        filename_format += ".{extension}"
+    kwargs['test_name']  = test_details['test_name']
+    kwargs['timestamp'] = datetime.strptime(test_details['test_start_time'],
+                                            LONG_DATE).strftime(SHORT_DATE)
+    return filename_format.format(**kwargs)
 
 def sanitize_options(options):
     """



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits