[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Many improvements to web_connectivity test



commit 7e1ee49e44480f62ac93a9f63aa90a64342df2ed
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Thu Apr 14 22:54:49 2016 +0200

    Many improvements to web_connectivity test
    
    * Run the probe resolver detection inside of the setupClass
    * Display a summary at the end of a test run summarising the results
    * Make the blocking detection logic more robust
---
 ooni/nettests/blocking/web_connectivity.py | 152 ++++++++++++++++++++---------
 1 file changed, 108 insertions(+), 44 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index d8276b3..320f614 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -8,6 +8,7 @@ from ipaddr import IPv4Address, AddressValueError
 from twisted.internet import reactor
 from twisted.internet.protocol import Factory, Protocol
 from twisted.internet.endpoints import TCP4ClientEndpoint
+from twisted.names import client, dns
 
 from twisted.internet import defer
 from twisted.python import usage
@@ -19,6 +20,17 @@ from ooni.utils.net import StringProducer, BodyReceiver
 from ooni.templates import httpt, dnst
 from ooni.errors import failureToString
 
+REQUEST_HEADERS = {
+    'User-Agent': ['Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, '
+                   'like Gecko) Chrome/47.0.2526.106 Safari/537.36'],
+    'Accept-Language': ['en-US;q=0.8,en;q=0.5'],
+    'Accept': ['text/html,application/xhtml+xml,application/xml;q=0.9,'
+               '*/*;q=0.8']
+}
+
+class InvalidControlResponse(Exception):
+    pass
+
 class TCPConnectProtocol(Protocol):
     def connectionMade(self):
         self.transport.loseConnection()
@@ -72,6 +84,21 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
     # Factor used to determine HTTP blockpage detection
     factor = 0.8
+    resolverIp = None
+
+    @classmethod
+    @defer.inlineCallbacks
+    def setUpClass(cls):
+        try:
+            answers = yield client.lookupAddress(
+                cls.localOptions['dns-discovery']
+            )
+            assert len(answers) > 0
+            assert len(answers[0]) > 0
+            cls.resolverIp = answers[0][0].payload.dottedQuad()
+        except Exception as exc:
+            log.exception(exc)
+            log.err("Failed to lookup the resolver IP address")
 
     def setUp(self):
         """
@@ -82,7 +109,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         if not self.input:
             raise Exception("No input specified")
 
-        self.report['client_resolver'] = None
+        self.report['client_resolver'] = self.resolverIp
         self.report['dns_consistency'] = None
         self.report['body_length_match'] = None
         self.report['accessible'] = None
@@ -111,9 +138,6 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
             }
         }
 
-    def dns_discovery(self):
-        return self.performALookup(self.localOptions['dns-discovery'])
-
     def experiment_dns_query(self):
         return self.performALookup(self.hostname)
 
@@ -161,11 +185,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         finished = defer.Deferred()
         response.deliverBody(BodyReceiver(finished, content_length))
         body = yield finished
-        self.control = json.loads(body)
+        try:
+            self.control = json.loads(body)
+            assert 'http_request' in self.control.keys()
+            assert 'tcp_connect' in self.control.keys()
+            assert 'dns' in self.control.keys()
+        except AssertionError, ValueError:
+            raise InvalidControlResponse(body)
         self.report['control'] = self.control
 
     def experiment_http_get_request(self):
-        return self.doRequest(self.input)
+        return self.doRequest(self.input, headers=REQUEST_HEADERS)
 
     def compare_body_lengths(self, experiment_http_response):
         control_body_length = self.control['http_request']['body_length']
@@ -233,27 +263,23 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         return success
 
     def determine_blocking(self, experiment_http_response, experiment_dns_answers):
-        blocking = None
+        blocking = False
         body_length_match = None
         dns_consistent = None
         tcp_connect = None
 
-        if self.report['control_failure'] is None and \
-                self.report['http_experiment_failure'] is None and \
-                self.report['control']['http_request']['failure'] is None:
+        if (self.report['http_experiment_failure'] is None and
+                    self.report['control']['http_request']['failure'] is None):
             body_length_match = self.compare_body_lengths(experiment_http_response)
 
-        if self.report['control_failure'] is None:
-            dns_consistent = self.compare_dns_experiments(experiment_dns_answers)
-
-        if self.report['control_failure'] is None:
-            tcp_connect = self.compare_tcp_experiments()
+        dns_consistent = self.compare_dns_experiments(experiment_dns_answers)
+        tcp_connect = self.compare_tcp_experiments()
 
         if dns_consistent == True and tcp_connect == False:
             blocking = 'tcp_ip'
 
-        elif dns_consistent == True and \
-                tcp_connect == True and body_length_match == False:
+        elif (dns_consistent == True and tcp_connect == True and
+                      body_length_match == False):
             blocking = 'http'
 
         elif dns_consistent == False:
@@ -270,28 +296,15 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         def dns_experiment_err(failure):
             self.report['dns_experiment_failure'] = failureToString(failure)
             return []
+        experiment_dns_answers = yield experiment_dns
 
-        results = yield defer.DeferredList([
-            self.dns_discovery(),
-            experiment_dns
-        ])
-
-        self.report['client_resolver'] = None
-        if results[0][0] == True:
-            self.report['client_resolver']  = results[0][1][0]
-
-        experiment_dns_answers = results[1][1]
         sockets = []
         for answer in experiment_dns_answers:
             if is_public_ipv4_address(answer) is True:
                 sockets.append("%s:80" % answer)
 
-        control_request = self.control_request(sockets)
-        @control_request.addErrback
-        def control_err(failure):
-            self.report['control_failure'] = failureToString(failure)
-
-        dl = [control_request]
+        # STEALTH in here we should make changes to make the test more stealth
+        dl = []
         for socket in sockets:
             dl.append(self.tcp_connect(socket))
         results = yield defer.DeferredList(dl)
@@ -303,20 +316,71 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
         experiment_http_response = yield experiment_http
 
-        blocking = self.determine_blocking(experiment_http_response, experiment_dns_answers)
-        self.report['blocking'] = blocking
+        control_request = self.control_request(sockets)
+        @control_request.addErrback
+        def control_err(failure):
+            log.err("Failed to perform control lookup")
+            self.report['control_failure'] = failureToString(failure)
+
+        yield control_request
 
-        if blocking is not None:
-            log.msg("%s: BLOCKING DETECTED due to %s" % (self.input, blocking))
+        if self.report['control_failure'] is None:
+            self.report['blocking'] = self.determine_blocking(experiment_http_response, experiment_dns_answers)
+
+        log.msg("")
+        log.msg("Result for %s" % self.input)
+        log.msg("-----------" + "-"*len(self.input))
+
+        if self.report['blocking'] is None:
+            log.msg("* Could not determine status of blocking due to "
+                    "failing control request")
+        elif self.report['blocking'] is False:
+            log.msg("* No blocking detected")
         else:
-            log.msg("%s: No blocking detected" % self.input)
+            log.msg("* BLOCKING DETECTED due to %s" % (self.report['blocking']))
 
-        if all(map(lambda x: x == None, [self.report['http_experiment_failure'],
-                                         self.report['dns_experiment_failure'],
-                                         blocking])):
-            log.msg("")
+        if (self.report['http_experiment_failure'] == None and
+                self.report['dns_experiment_failure'] == None and
+                self.report['blocking'] in (False, None)):
             self.report['accessible'] = True
-            log.msg("%s: is accessible" % self.input)
+            log.msg("* Is accessible")
         else:
-            log.msg("%s: is NOT accessible" % self.input)
+            log.msg("* Is NOT accessible")
             self.report['accessible'] = False
+
+    def postProcessor(self, measurements):
+        self.summary['accessible'] = self.summary.get('accessible', [])
+        self.summary['not-accessible'] = self.summary.get('not-accessible', [])
+        self.summary['blocked'] = self.summary.get('blocked', [])
+
+        if self.report['blocking'] not in (False, None):
+            self.summary['blocked'].append((self.input,
+                                            self.report['blocking']))
+        if self.report['accessible'] is True:
+            self.summary['accessible'].append(self.input)
+        else:
+            self.summary['not-accessible'].append(self.input)
+        return self.report
+
+    def displaySummary(self, summary):
+
+        if len(summary['accessible']) > 0:
+            log.msg("")
+            log.msg("Accessible URLS")
+            log.msg("---------------")
+            for url in summary['accessible']:
+                log.msg("* {}".format(url))
+
+        if len(summary['not-accessible']) > 0:
+            log.msg("")
+            log.msg("Not accessible URLS")
+            log.msg("---------------")
+            for url in summary['not-accessible']:
+                log.msg("* {}".format(url))
+
+        if len(summary['blocked']) > 0:
+            log.msg("")
+            log.msg("Blocked URLS")
+            log.msg("------------")
+            for url, reason in summary['blocked']:
+                log.msg("* {} due to {}".format(url, reason))



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits