[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Add a configuration option that allows to prioritise non-onion backends



commit 9d52e351e79521175ce2e91e5b00dbe72d8f1814
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Thu Jun 16 20:03:57 2016 +0300

    Add a configuration option that allows to prioritise non-onion backends
    
    This option allows a user to specify that they wish to use either a https,
    cloudfronted or onion backend server.
    
    Write unittests to use the priority address
---
 data/ooniprobe.conf.sample |   3 ++
 ooni/backend_client.py     |  26 +++++------
 ooni/constants.py          |   5 +++
 ooni/deck.py               | 109 ++++++++++++++++++++++++++-------------------
 ooni/errors.py             |   4 ++
 ooni/oonicli.py            |  11 ++---
 ooni/tests/bases.py        |  10 +++--
 ooni/tests/mocks.py        |   9 ++++
 ooni/tests/test_deck.py    |  70 ++++++++++++++++++++++++++---
 9 files changed, 175 insertions(+), 72 deletions(-)

diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index be35364..e310ddc 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -48,7 +48,10 @@ advanced:
     report_log_file: null
     inputs_dir: null
     decks_dir: null
+    # If we should support communicating to plaintext backends (via HTTP)
     insecure_backend: false
+    # The preferred backend type, can be one of onion, https or cloudfront
+    preferred_backend: onion
 tor:
     #socks_port: 8801
     #control_port: 8802
diff --git a/ooni/backend_client.py b/ooni/backend_client.py
index 0e85dd7..71b774f 100644
--- a/ooni/backend_client.py
+++ b/ooni/backend_client.py
@@ -19,6 +19,18 @@ from ooni.utils.net import BodyReceiver, StringProducer, Downloader
 from ooni.utils.socks import TrueHeadersSOCKS5Agent
 
 
+def guess_backend_type(address):
+    if address is None:
+        raise e.InvalidAddress
+    if onion.is_onion_address(address):
+        return 'onion'
+    elif address.startswith('https://'):
+        return 'https'
+    elif address.startswith('http://'):
+        return 'http'
+    else:
+        raise e.InvalidAddress
+
 class OONIBClient(object):
     def __init__(self, address=None, settings={}):
         self.base_headers = {}
@@ -26,7 +38,7 @@ class OONIBClient(object):
         self.base_address = settings.get('address', address)
 
         if self.backend_type is None:
-            self._guessBackendType()
+            self.backend_type = guess_backend_type(self.base_address)
         self.backend_type = self.backend_type.encode('ascii')
 
         if self.backend_type == 'cloudfront':
@@ -39,18 +51,6 @@ class OONIBClient(object):
             'front': settings.get('front', '').encode('ascii')
         }
 
-    def _guessBackendType(self):
-        if self.base_address is None:
-            raise e.InvalidAddress
-        if onion.is_onion_address(self.base_address):
-            self.backend_type = 'onion'
-        elif self.base_address.startswith('https://'):
-            self.backend_type = 'https'
-        elif self.base_address.startswith('http://'):
-            self.backend_type = 'http'
-        else:
-            raise e.InvalidAddress
-
     def _setupBaseAddress(self):
         parsed_address = urlparse(self.base_address)
         if self.backend_type == 'onion':
diff --git a/ooni/constants.py b/ooni/constants.py
index 882edb4..ef1f3d7 100644
--- a/ooni/constants.py
+++ b/ooni/constants.py
@@ -1,4 +1,9 @@
 CANONICAL_BOUNCER_ONION = 'httpo://nkvphnp3p6agi5qq.onion'
+CANONICAL_BOUNCER_HTTPS = 'https://bouncer.ooni.io'
+CANONICAL_BOUNCER_CLOUDFRONT = (
+    'XXXX.cloudfront.net',
+    'https://a0.awsstatic.com/'
+)
 
 MEEK_BRIDGES = [
     ("meek 0.0.2.0:2 B9E7141C594AF25699E0079C1F0146F409495296 "
diff --git a/ooni/deck.py b/ooni/deck.py
index 82f98a1..2aa958d 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
 
 from ooni.backend_client import CollectorClient, BouncerClient
-from ooni.backend_client import WebConnectivityClient
+from ooni.backend_client import WebConnectivityClient, guess_backend_type
 from ooni.nettest import NetTestLoader
 from ooni.settings import config
 from ooni.utils import log, onion
+from ooni import constants
 from ooni import errors as e
 
 from twisted.python.filepath import FilePath
@@ -120,7 +121,29 @@ class Deck(InputFile):
                  no_collector=False):
         self.id = deck_hash
         self.no_collector = no_collector
-        self.bouncer = bouncer
+
+        self.preferred_backend = config.advanced.get(
+            "preferred_backend", "onion"
+        )
+        if self.preferred_backend not in ["onion", "https", "cloudfront"]:
+            raise e.InvalidPreferredBackend
+
+        if bouncer is None:
+            bouncer_address = getattr(
+                constants, "CANONICAL_BOUNCER_{0}".format(
+                    self.preferred_backend.upper()
+                )
+            )
+            if self.preferred_backend == "cloudfront":
+                self.bouncer = self._BouncerClient(settings={
+                    'address': bouncer_address[0],
+                    'front': bouncer_address[1],
+                    'type': 'cloudfront'
+                })
+            else:
+                self.bouncer = self._BouncerClient(bouncer_address)
+        else:
+            self.bouncer = self._BouncerClient(bouncer)
 
         self.requiresTor = False
 
@@ -167,19 +190,23 @@ class Deck(InputFile):
                     collector_address
                 )
             if test['options'].get('bouncer', None) is not None:
-                self.bouncer = test['options']['bouncer']
+                self.bouncer = self._BouncerClient(test['options']['bouncer'])
+                if self.bouncer.backend_type is "onion":
+                    self.requiresTor = True
             self.insert(net_test_loader)
 
     def insert(self, net_test_loader):
         """ Add a NetTestLoader to this test deck """
+        if (net_test_loader.collector is not None
+                and net_test_loader.collector.backend_type is "onion"):
+            self.requiresTor = True
         try:
             net_test_loader.checkOptions()
             if net_test_loader.requiresTor:
                 self.requiresTor = True
         except e.MissingTestHelper:
-            if not self.bouncer:
-                raise
-            self.requiresTor = True
+            if self.preferred_backend is "onion":
+                self.requiresTor = True
 
         self.netTestLoaders.append(net_test_loader)
 
@@ -188,6 +215,7 @@ class Deck(InputFile):
         """ fetch and verify inputs for all NetTests in the deck """
         log.msg("Fetching required net test inputs...")
         for net_test_loader in self.netTestLoaders:
+            # XXX figure out if we want to keep this or drop this.
             yield self.fetchAndVerifyNetTestInput(net_test_loader)
 
         if self.bouncer:
@@ -196,44 +224,34 @@ class Deck(InputFile):
 
 
     def sortAddressesByPriority(self, priority_address, alternate_addresses):
-        onion_addresses= []
-        cloudfront_addresses= []
-        https_addresses = []
-        plaintext_addresses = []
-
-        if onion.is_onion_address(priority_address):
-            priority_address = {
-                'address': priority_address,
-                'type': 'onion'
-            }
-        elif priority_address.startswith('https://'):
-            priority_address = {
-                'address': priority_address,
-                'type': 'https'
-            }
-        elif priority_address.startswith('http://'):
-            priority_address = {
-                'address': priority_address,
-                'type': 'http'
-            }
-        else:
-            raise e.InvalidOONIBCollectorAddress
+        prioritised_addresses = []
+
+        backend_type = guess_backend_type(priority_address)
+        priority_address = {
+            'address': priority_address,
+            'type': backend_type
+        }
+        address_priority = ['onion', 'https', 'cloudfront', 'http']
+        address_priority.remove(self.preferred_backend)
+        address_priority.insert(0, self.preferred_backend)
 
         def filter_by_type(collectors, collector_type):
-            return filter(lambda x: x['type'] == collector_type,
-                          collectors)
-        onion_addresses += filter_by_type(alternate_addresses, 'onion')
-        https_addresses += filter_by_type(alternate_addresses, 'https')
-        cloudfront_addresses += filter_by_type(alternate_addresses,
-                                                'cloudfront')
+            return filter(lambda x: x['type'] == collector_type, collectors)
+
+        if (priority_address['type'] != self.preferred_backend):
+            valid_alternatives = filter_by_type(alternate_addresses,
+                                                self.preferred_backend)
+            if len(valid_alternatives) > 0:
+                alternate_addresses += [priority_address]
+                priority_address = valid_alternatives[0]
+                alternate_addresses.remove(priority_address)
 
-        plaintext_addresses += filter_by_type(alternate_addresses, 'http')
+        prioritised_addresses += [priority_address]
+        for address_type in address_priority:
+            prioritised_addresses += filter_by_type(alternate_addresses,
+                                                    address_type)
 
-        return ([priority_address] +
-                onion_addresses +
-                https_addresses +
-                cloudfront_addresses +
-                plaintext_addresses)
+        return prioritised_addresses
 
     @defer.inlineCallbacks
     def getReachableCollector(self, collector_address, collector_alternate):
@@ -289,9 +307,12 @@ class Deck(InputFile):
     @defer.inlineCallbacks
     def getReachableTestHelpersAndCollectors(self, net_tests):
         for net_test in net_tests:
+
+            primary_address = net_test['collector']
+            alternate_addresses = net_test.get('collector-alternate', [])
             net_test['collector'] = yield self.getReachableCollector(
-                        net_test['collector'],
-                        net_test.get('collector-alternate', [])
+                        primary_address,
+                        alternate_addresses
             )
 
             for test_helper_name, test_helper_address in net_test['test-helpers'].items():
@@ -307,8 +328,6 @@ class Deck(InputFile):
 
     @defer.inlineCallbacks
     def lookupCollectorAndTestHelpers(self):
-        oonibclient = self._BouncerClient(self.bouncer)
-
         required_nettests = []
 
         requires_test_helpers = False
@@ -333,7 +352,7 @@ class Deck(InputFile):
         if not requires_test_helpers and not requires_collector:
             defer.returnValue(None)
 
-        response = yield oonibclient.lookupTestCollector(required_nettests)
+        response = yield self.bouncer.lookupTestCollector(required_nettests)
         try:
             provided_net_tests = yield self.getReachableTestHelpersAndCollectors(response['net-tests'])
         except e.NoReachableCollectors:
diff --git a/ooni/errors.py b/ooni/errors.py
index 8197f34..faa3627 100644
--- a/ooni/errors.py
+++ b/ooni/errors.py
@@ -316,3 +316,7 @@ class NoReachableCollectors(Exception):
 
 class NoReachableTestHelpers(Exception):
     pass
+
+
+class InvalidPreferredBackend(Exception):
+    pass
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 74bf9ac..43654b6 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -45,9 +45,9 @@ class Options(usage.Options):
         ["collector", "c", None, "Specify the address of the collector for "
                                  "test results. In most cases a user will "
                                  "prefer to specify a bouncer over this."],
-        ["bouncer", "b", CANONICAL_BOUNCER_ONION, "Specify the bouncer used to "
-                                            "obtain the address of the "
-                                            "collector and test helpers."],
+        ["bouncer", "b", None, "Specify the bouncer used to "
+                               "obtain the address of the "
+                               "collector and test helpers."],
         ["logfile", "l", None, "Write to this logs to this filename."],
         ["pcapfile", "O", None, "Write a PCAP of the ooniprobe session to "
                                 "this filename."],
@@ -388,10 +388,11 @@ def runWithDirector(global_options):
         log.msg("Not reporting using a collector")
         global_options['collector'] = None
         start_tor = False
-    else:
+    elif config.advanced.get("preferred_backend", "onion") == "onion":
         start_tor = True
 
-    if global_options['collector']:
+    if (global_options['collector'] and
+            config.advanced.get("preferred_backend", "onion") == "onion"):
         start_tor |= True
 
     return runTestWithDirector(director=director,
diff --git a/ooni/tests/bases.py b/ooni/tests/bases.py
index 5d177fa..31cbf94 100644
--- a/ooni/tests/bases.py
+++ b/ooni/tests/bases.py
@@ -1,3 +1,5 @@
+import os
+import shutil
 from twisted.trial import unittest
 
 from ooni.settings import config
@@ -5,11 +7,13 @@ from ooni.settings import config
 
 class ConfigTestCase(unittest.TestCase):
     def setUp(self):
-        config.initialize_ooni_home("ooni_home")
+        self.ooni_home_dir = os.path.abspath("ooni_home")
+        self.config = config
+        self.config.initialize_ooni_home("ooni_home")
+        super(ConfigTestCase, self).setUp()
 
     def skipTest(self, reason):
         raise unittest.SkipTest(reason)
 
     def tearDown(self):
-        config.set_paths()
-        config.read_config_file()
+        shutil.rmtree("ooni_home")
diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py
index db7b154..a377fba 100644
--- a/ooni/tests/mocks.py
+++ b/ooni/tests/mocks.py
@@ -222,6 +222,15 @@ class MockBouncerClient(object):
                 'version': net_test['version'],
                 'input-hashes': net_test['input-hashes'],
                 'collector': 'httpo://thirteenchars123.onion',
+                'collector-alternate': [
+                    {'type': 'https', 'address': 'https://collector.ooni.io'},
+                    {'type': 'http', 'address': 'http://collector.ooni.io'},
+                    {
+                        'type': 'cloudfront',
+                        'address': 'https://address.cloudfront.net',
+                        'front': 'https://front.cloudfront.net'
+                    },
+                ],
                 'test-helpers': test_helpers
             })
         return defer.succeed(ret)
diff --git a/ooni/tests/test_deck.py b/ooni/tests/test_deck.py
index e4f661f..f299e70 100644
--- a/ooni/tests/test_deck.py
+++ b/ooni/tests/test_deck.py
@@ -6,8 +6,11 @@ from twisted.trial import unittest
 from hashlib import sha256
 from ooni import errors
 from ooni.deck import InputFile, Deck, nettest_to_path
+from ooni.tests.bases import ConfigTestCase
 from ooni.tests.mocks import MockBouncerClient, MockCollectorClient
 
+FAKE_BOUNCER_ADDRESS = "httpo://thirteenchars123.onion"
+
 net_test_string = """
 from twisted.python import usage
 from ooni.nettest import NetTestCase
@@ -71,7 +74,7 @@ class BaseTestCase(unittest.TestCase):
             test_file: manipulation/http_invalid_request_line
             testdeck: null
 """
-
+        super(BaseTestCase, self).setUp()
 
 
 class TestInputFile(BaseTestCase):
@@ -112,7 +115,7 @@ class TestInputFile(BaseTestCase):
         assert input_file.descriptorCached
 
 
-class TestDeck(BaseTestCase):
+class TestDeck(BaseTestCase, ConfigTestCase):
     def setUp(self):
         super(TestDeck, self).setUp()
         deck_hash = sha256(self.dummy_deck_content).hexdigest()
@@ -127,9 +130,10 @@ class TestDeck(BaseTestCase):
         os.remove(self.deck_file)
         if self.filename != "":
             os.remove(self.filename)
+        super(TestDeck, self).tearDown()
 
     def test_open_deck(self):
-        deck = Deck(bouncer="httpo://foo.onion",
+        deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
                     decks_directory=".")
         deck.loadDeck(self.deck_file)
         assert len(deck.netTestLoaders) == 1
@@ -139,7 +143,7 @@ class TestDeck(BaseTestCase):
             "annotations": {"spam": "ham"},
             "collector": "httpo://thirteenchars123.onion"
         }
-        deck = Deck(bouncer="httpo://foo.onion",
+        deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
                     decks_directory=".")
         deck.loadDeck(self.deck_file,
                       global_options=global_options)
@@ -153,7 +157,7 @@ class TestDeck(BaseTestCase):
         )
 
     def test_save_deck_descriptor(self):
-        deck = Deck(bouncer="httpo://foo.onion",
+        deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
                     decks_directory=".")
         deck.loadDeck(self.deck_file)
         deck.load({'name': 'spam',
@@ -169,8 +173,9 @@ class TestDeck(BaseTestCase):
 
     @defer.inlineCallbacks
     def test_lookup_test_helpers_and_collector(self):
-        deck = Deck(bouncer="httpo://foo.onion",
+        deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
                     decks_directory=".")
+        deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
         deck._BouncerClient = MockBouncerClient
         deck._CollectorClient = MockCollectorClient
         deck.loadDeck(self.deck_file)
@@ -211,3 +216,56 @@ class TestDeck(BaseTestCase):
         self.assertRaises(errors.NetTestNotFound,
                           nettest_to_path,
                           "invalid_test")
+
+    @defer.inlineCallbacks
+    def test_lookup_test_helpers_and_collector_cloudfront(self):
+        self.config.advanced.preferred_backend = "cloudfront"
+        deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
+                    decks_directory=".")
+        deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
+        deck._BouncerClient = MockBouncerClient
+        deck._CollectorClient = MockCollectorClient
+        deck.loadDeck(self.deck_file)
+
+        self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
+
+        yield deck.lookupCollectorAndTestHelpers()
+
+        self.assertEqual(
+            deck.netTestLoaders[0].collector.settings['address'],
+            'https://address.cloudfront.net'
+        )
+        self.assertEqual(
+            deck.netTestLoaders[0].collector.settings['front'],
+            'https://front.cloudfront.net'
+        )
+
+        self.assertEqual(
+            deck.netTestLoaders[0].localOptions['backend'],
+            '127.0.0.1'
+        )
+
+
+    @defer.inlineCallbacks
+    def test_lookup_test_helpers_and_collector_https(self):
+        self.config.advanced.preferred_backend = "https"
+        deck = Deck(bouncer=FAKE_BOUNCER_ADDRESS,
+                    decks_directory=".")
+        deck.bouncer = MockBouncerClient(FAKE_BOUNCER_ADDRESS)
+        deck._BouncerClient = MockBouncerClient
+        deck._CollectorClient = MockCollectorClient
+        deck.loadDeck(self.deck_file)
+
+        self.assertEqual(len(deck.netTestLoaders[0].missingTestHelpers), 1)
+
+        yield deck.lookupCollectorAndTestHelpers()
+
+        self.assertEqual(
+            deck.netTestLoaders[0].collector.settings['address'],
+            'https://collector.ooni.io'
+        )
+
+        self.assertEqual(
+            deck.netTestLoaders[0].localOptions['backend'],
+            '127.0.0.1'
+        )



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits