[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [oonib/master] Implement policy aware bouncing support in the backend



commit fdc0a4460cb3df9ec7ab55669e71f2008e0ee322
Author: kudrom <kudrom@xxxxxxxxxx>
Date:   Mon Aug 25 17:19:38 2014 +0200

    Implement policy aware bouncing support in the backend
---
 data/bouncer.yaml         |   17 +++++++-
 oonib/bouncer/handlers.py |  102 +++++++++++++++++++++++++++++++++++----------
 oonib/errors.py           |   10 +++++
 oonib/handlers.py         |    3 +-
 4 files changed, 108 insertions(+), 24 deletions(-)

diff --git a/data/bouncer.yaml b/data/bouncer.yaml
index 9ab2dc0..acf2a9a 100644
--- a/data/bouncer.yaml
+++ b/data/bouncer.yaml
@@ -1,3 +1,18 @@
 collector:
   httpo://ihiderha53f36lsd.onion:
-      test-helper: {dns: '93.95.227.200:57004', http-return-json-headers: 'http://93.95.227.200', ssl: 'https://93.95.227.20', tcp-echo: '213.138.109.232', traceroute: '93.95.227.200'}
+    policy:
+      input:
+      - {id: 37e60e13536f6afe47a830bfb6b371b5cf65da66d7ad65137344679b24fdccd1}
+      - {id: e0611ecd28bead38a7afeb4dda8ae3449d0fc2e1ba53fa7355f2799dce9af290}
+      nettest:
+      - {name: dns_consistency, version: '0.5'}
+      - {name: http_requests_test, version: 0.2.3}
+      - {name: tcp_connect, version: '0.1'}
+      - {name: captivep, version: '0.2'}
+      - {name: daphn3, version: '0.1'}
+      - {name: dns_spoof, version: '0.2'}
+      - {name: http_header_field_manipulation, version: 0.1.3}
+      - {name: http_invalid_request_line, version: 0.1.4}
+      - {name: multiprotocol_traceroute_test, version: 0.1.1}
+    test-helper: {dns: '93.95.227.200:57004', http-return-json-headers: 'http://93.95.227.200',
+      ssl: 'https://93.95.227.20', tcp-echo: 213.138.109.232, traceroute: 93.95.227.200}
diff --git a/oonib/bouncer/handlers.py b/oonib/bouncer/handlers.py
index 81d13f3..c49338e 100644
--- a/oonib/bouncer/handlers.py
+++ b/oonib/bouncer/handlers.py
@@ -7,24 +7,30 @@ from oonib.config import config
 
 
 class Bouncer(object):
-    def __init__(self):
-        with open(config.main.bouncer_file) as f:
-            bouncerFile = yaml.safe_load(f)
-        self.updateKnownHelpers(bouncerFile)
-        self.updateKnownCollectors(bouncerFile)
 
-    def updateKnownCollectors(self, bouncerFile):
+    def __init__(self, bouncer_file):
+        with open(bouncer_file) as f:
+            self.bouncerFile = yaml.safe_load(f)
+        self.updateKnownHelpers()
+        self.updateKnownCollectors()
+
+    def updateKnownCollectors(self):
         """
         Initialize the list of all known collectors
         """
-        self.knownCollectors = []
-        for collectorName, helpers in bouncerFile['collector'].items():
-            if collectorName not in self.knownCollectors:
-                self.knownCollectors.append(collectorName)
-
-    def updateKnownHelpers(self, bouncerFile):
+        self.knownCollectorsWithPolicy = []
+        self.knownCollectorsWithoutPolicy = []
+        for collectorName, content in self.bouncerFile['collector'].items():
+            if content.get('policy') is not None and \
+                    collectorName not in self.knownCollectorsWithPolicy:
+                self.knownCollectorsWithPolicy.append(collectorName)
+            elif content.get('policy') is None and \
+                    collectorName not in self.knownCollectorsWithoutPolicy:
+                self.knownCollectorsWithoutPolicy.append(collectorName)
+
+    def updateKnownHelpers(self):
         self.knownHelpers = {}
-        for collectorName, helpers in bouncerFile['collector'].items():
+        for collectorName, helpers in self.bouncerFile['collector'].items():
             for helperName, helperAddress in helpers['test-helper'].items():
                 if helperName not in self.knownHelpers.keys():
                     self.knownHelpers[helperName] = []
@@ -106,14 +112,63 @@ class Bouncer(object):
                 response = {'error': 'test-helper-not-found'}
                 return response
 
-        response['default'] = {'collector':
-                               random.choice(self.knownCollectors)}
+        if len(self.knownCollectorsWithoutPolicy) > 0:
+            default_collector = random.choice(
+                self.knownCollectorsWithoutPolicy)
+        else:
+            default_collector = None
+        response['default'] = {'collector': default_collector}
         return response
 
+    def collectorAccepting(self, net_test_name, input_hashes, test_helpers):
+        for collector_address in self.knownCollectorsWithPolicy:
+            collector = self.bouncerFile['collector'][collector_address]
+            supported_net_tests = [x['name'] for x in collector['policy']['nettest']]
+            supported_input_hashes = [x['id'] for x in collector['policy']['input']]
+            if net_test_name not in supported_net_tests:
+                continue
+            if any([input_hash not in supported_input_hashes for input_hash in input_hashes]):
+                continue
+            if all([x in collector['test-helper'].keys() for x in test_helpers]):
+                return collector_address
+        if len(self.knownCollectorsWithoutPolicy) > 0:
+            return random.choice(self.knownCollectorsWithoutPolicy)
+        else:
+            raise e.CollectorNotFound
+
+    def filterByNetTests(self, requested_nettests):
+        """
+        Here we will return a list containing test helpers and collectors for
+        the required nettests.
+        We give favour to the collectors that have a stricter policy and if
+        those fail we will resort to using the collectors with a more lax
+        policy.
+        """
+        nettests = []
+        for requested_nettest in requested_nettests:
+            collector = self.collectorAccepting(
+                requested_nettest['name'],
+                requested_nettest['input-hashes'],
+                requested_nettest['test-helpers'])
+            test_helpers = {}
+            for test_helper in requested_nettest['test-helpers']:
+                test_helpers[test_helper] = self.bouncerFile['collector'][collector]['test-helper'][test_helper]
+
+            nettest = {
+                'name': requested_nettest['name'],
+                'version': requested_nettest['version'],
+                'input-hashes': requested_nettest['input-hashes'],
+                'test-helpers': test_helpers,
+                'collector': collector,
+            }
+            nettests.append(nettest)
+        return {'net-tests': nettests}
+
 
 class BouncerQueryHandler(OONIBHandler):
+
     def initialize(self):
-        self.bouncer = Bouncer()
+        self.bouncer = Bouncer(config.main.bouncer_file)
 
     def post(self):
         try:
@@ -121,13 +176,16 @@ class BouncerQueryHandler(OONIBHandler):
         except ValueError:
             raise e.InvalidRequest
 
-        try:
+        if 'test-helpers' in query:
             requested_helpers = query['test-helpers']
-        except KeyError:
-            raise e.TestHelpersKeyMissing
+            if not isinstance(requested_helpers, list):
+                raise e.InvalidRequest
+            response = self.bouncer.filterHelperAddresses(requested_helpers)
 
-        if not isinstance(requested_helpers, list):
-            raise e.InvalidRequest
+        elif 'net-tests' in query:
+            response = self.bouncer.filterByNetTests(query['net-tests'])
+
+        else:
+            raise e.TestHelpersOrNetTestsKeyMissing
 
-        response = self.bouncer.filterHelperAddresses(requested_helpers)
         self.write(response)
diff --git a/oonib/errors.py b/oonib/errors.py
index 3f1167a..e20d471 100644
--- a/oonib/errors.py
+++ b/oonib/errors.py
@@ -83,6 +83,11 @@ class ReportNotFound(OONIBError):
     log_message = "report-not-found"
 
 
+class CollectorNotFound(OONIBError):
+    status_code = 404
+    log_message = "collector-not-found"
+
+
 class NoValidCollector(OONIBError):
     pass
 
@@ -92,6 +97,11 @@ class TestHelpersKeyMissing(OONIBError):
     log_message = "test-helpers-key-missing"
 
 
+class TestHelpersOrNetTestsKeyMissing(OONIBError):
+    status_code = 400
+    log_message = "test-helpers-or-net-test-key-missing"
+
+
 class TestHelperNotFound(OONIBError):
     status_code = 404
     log_message = "test-helper-not-found"
diff --git a/oonib/handlers.py b/oonib/handlers.py
index bbf8ade..1417cc5 100644
--- a/oonib/handlers.py
+++ b/oonib/handlers.py
@@ -7,6 +7,7 @@ from oonib import log
 
 
 class OONIBHandler(web.RequestHandler):
+
     def write_error(self, status_code, exception=None, **kw):
         self.set_status(status_code)
         if hasattr(exception, 'log_message') and exception.log_message is not None:
@@ -14,7 +15,7 @@ class OONIBHandler(web.RequestHandler):
         elif 400 <= status_code < 600:
             self.write({'error': status_code})
         else:
-            log.error(exception)
+            log.exception(exception)
             self.write({'error': 'error'})
 
     def write(self, chunk):



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits