[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Hardcoded vendor tests in separate function which optionally runs through setting in config, user-defined tests are optionally specified in asset file.



commit 6d57e84a19699a6f0dd2c6152ecafc87edacec9e
Author: Isis Lovecruft <isis@xxxxxxxxxxxxxxxxxxxxx>
Date:   Mon Apr 9 19:32:12 2012 -0700

    Hardcoded vendor tests in separate function which optionally runs through setting in config, user-defined tests are optionally specified in asset file.
---
 assets/captive_portal_tests.txt |    4 +
 ooni-probe.conf                 |   13 ++-
 tests/captiveportal.py          |  208 ++++++++++++++++++++++++++++-----------
 3 files changed, 166 insertions(+), 59 deletions(-)

diff --git a/assets/captive_portal_tests.txt b/assets/captive_portal_tests.txt
new file mode 100644
index 0000000..1bd016f
--- /dev/null
+++ b/assets/captive_portal_tests.txt
@@ -0,0 +1,4 @@
+
+http://ooni.nu, Open Observatory of Network Interference, 200
+http://www.patternsinthevoid.net/2CDB8B35pub.asc, mQINBE5qkHABEADVnasCm9w9hUff1E4iKnzcAdp4lx6XU5USmYdwKg2RQt2VFqWQ, 200
+http://www.google.com, Search the world's information, 200
diff --git a/ooni-probe.conf b/ooni-probe.conf
index a744fcf..a19c1d0 100644
--- a/ooni-probe.conf
+++ b/ooni-probe.conf
@@ -40,13 +40,20 @@ dns_reverse_lookup = true
 
 ### captiveportal testing configuration parameters
 
+# This is an optional list of user defined captive portal tests, 
+# one per line, with each line in the format: 
+# experiment_url, control_result, control_code 
+# where experiment_url is the test page to retrieve,
+#       control_result is some unique text found on the test page,
+#   and control_code is the expected HTTP status code.
+captive_portal = captive_portal_tests.txt  
+
 # The default User Agent that ooni-probe should send for
 # HTTP requests (pretend we're a Windows box running FF10): 
 default_ua = Mozilla/5.0 (Windows NT 6.1; WOW64; rv:10.0.2) Gecko/20100101 Firefox/10.0.2
 
-# This is the list of captive portal tests, in the format:
-# test_name, experiment_url, control_result, control_code
-captive_portal = captive_portal_tests.txt  
+# Enable vendor tests for captive portals:
+do_captive_portal_vendor_tests = true
 
 ### traceroute testing related config parameters
 
diff --git a/tests/captiveportal.py b/tests/captiveportal.py
index 324d3cc..494a4c3 100644
--- a/tests/captiveportal.py
+++ b/tests/captiveportal.py
@@ -23,7 +23,7 @@ __desc__ = "Captive portal detection test"
 
 class CaptivePortalAsset(Asset):
     """
-    Parses captive_portal_test.txt into an Asset.
+    Parses captive_portal_tests.txt into an Asset.
     """
     def __init__(self, file=None):
         self = Asset.__init__(self, file)
@@ -32,28 +32,13 @@ class CaptivePortalAsset(Asset):
         self = Asset.parse_line(self, line)
         return line.replace('\n', '').split(', ')
 
-    '''
-    def next_asset(self):
-        self = Asset.next_asset(self)
-        with self.fh as fh:
-            asset_list = []
-            lines = fh.readlines()
-            for line in lines:
-                parsed_line = self.parse_line(line)
-                if parsed_line:
-                    asset_list.append(parsed_line)
-                else:
-                    fh.seek(0)
-                    raise StopIteration
-            return asset_list
-    '''
-
 class CaptivePortal(Test):
     """
     Compares content and status codes of HTTP responses, and attempts
     to determine if content has been altered.
 
-    TODO: compare headers
+    TODO: compare headers, random URL requests with control obtained
+    through Tor.
     """
     def __init__(self, ooni):
         Test.__init__(self, ooni, name='test')
@@ -70,7 +55,7 @@ class CaptivePortal(Test):
         return response
  
     def http_content_match_fuzzy_opt(self, experimental_url, control_result,
-                                     fuzzy=False):
+                                     headers=None, fuzzy=False):
         """
         Makes an HTTP request on port 80 for experimental_url, then
         compares the response_content of experimental_url with the
@@ -78,13 +63,15 @@ class CaptivePortal(Test):
         True, the response_content is compared with a regex of the
         control_result. If the response_content from the
         experimental_url and the control_result match, returns True
-        with the HTTP status code, False if otherwise.
+        with the HTTP status code, False and status code if otherwise.
         """
         log = self.logger
-        default_ua = self.default_ua
 
-        response = self.http_fetch(experimental_url, 
-                                   headers={'User-Agent': default_ua})
+        if headers is None:
+            default_ua = self.default_ua
+            headers = {'User-Agent': default_ua}
+
+        response = self.http_fetch(experimental_url, headers)
         response_content = response.read()
         response_code = response.code
         if response_content is not None:
@@ -129,6 +116,89 @@ class CaptivePortal(Test):
             return False
         return True
 
+    def run_vendor_tests(self, *a, **kw):
+        """
+        These are several vendor tests used to detect the presence of
+        a captive portal. Each test compares HTTP status code and
+        content to the control results and has its own User-Agent
+        string, in order to emulate the test as it would occur on the
+        device it was intended for. Vendor tests are defined in the
+        format: 
+        [experimental_url, control_response, control_code, ua, test_name]
+        """
+        cm = self.http_content_match_fuzzy_opt
+        sm = self.http_status_code_match
+        snm = self.http_status_code_no_match
+        
+        log = self.logger
+
+        vendor_tests = [['http://www.apple.com/library/test/success.html',
+                         'Success',
+                         '200',
+                         'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543a Safari/419.3',
+                         'Apple HTTP Captive Portal'],
+                        ['http://tools.ietf.org/html/draft-nottingham-http-portal-02',
+                         '428 Network Authentication Required',
+                         '428',
+                         'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0',
+                         'W3 Captive Portal'],
+                        ['http://www.msftncsi.com/ncsi.txt',
+                         'Microsoft NCSI',
+                         '200',
+                         'Microsoft NCSI',
+                         'MS HTTP Captive Portal',]]
+
+        log.debug("Getting vendor test data")
+        
+        for vt in vendor_tests:
+            experiment_url = vt[0]
+            control_result = vt[1]
+            control_code = vt[2]
+            ua = vt[3]
+            test_name = vt[4]
+
+            if test_name == "MS HTTP Captive Portal":
+                log.info("Running the %s test..." % test_name)
+                content_match, experiment_code = cm(experiment_url, control_result,
+                                                    headers={'User-Agent': ua})
+                status_match = sm(experiment_code, control_code)
+                if status_match and content_match:
+                    log.info("The %s test was unable to detect a captive portal."
+                             % test_name)
+                else:
+                    log.info("The %s test shows that your network is filtered." 
+                             % test_name)
+                
+            elif test_name == "Apple HTTP Captive Portal":
+                log.info("Running the %s test..." % test_name)
+                content_fuzzy_match, experiment_code = cm(experiment_url, 
+                                                          control_result,
+                                                          headers={'User-Agent': ua},
+                                                          fuzzy=True)
+                status_match = sm(experiment_code, control_code)
+                if status_match and content_fuzzy_match:
+                    log.info("The %s test was unable to detect a captive portal."
+                             % test_name)
+                else:
+                    log.info("The %s test shows that your network is filtered." 
+                             % test_name)
+                
+            elif test_name == "W3 Captive Portal":
+                log.info("Running the %s test..." % test_name)
+                content_fuzzy_match, experiment_code = cm(experiment_url, 
+                                                          control_result,
+                                                          headers={'User-Agent': ua},
+                                                          fuzzy=True)
+                status_no_match = snm(experiment_code, control_code)
+                if status_no_match and content_fuzzy_match:
+                    log.info("The %s test was unable to detect a captive portal."
+                             % test_name)
+                else:
+                    log.info("The %s test shows that your network is filtered." 
+                             % test_name)
+            else:
+                log.warn("Ooni is trying to run an undefined CP vendor test.")
+
     def experiment(self, *a, **kw):
         """
         Compares the content and status code of the HTTP response for
@@ -139,53 +209,79 @@ class CaptivePortal(Test):
         in the experimental content. Returns True if matches are found,
         and False if otherwise.
         """
-        test_name = kw['data'][0] 
-        experiment_url = kw['data'][1]
-        control_result = kw['data'][2]
-        control_code = kw['data'][3]
-
+        if (os.path.isfile(os.path.join(self.config.main.assetdir,
+                                        self.config.tests.captive_portal))):
+            kw['data'].append(None)
+            kw['data'].append('user defined')
+        
+        experiment_url = kw['data'][0]
+        control_result = kw['data'][1]
+        control_code = kw['data'][2]
+        ua = kw['data'][3]
+        test_name = kw['data'][4]
+    
         cm = self.http_content_match_fuzzy_opt
         sm = self.http_status_code_match
-
+        snm = self.http_status_code_no_match
+        
         log = self.logger
-        log.info("Running the %s test..." % test_name)
         
-        content_match, experiment_code = cm(experiment_url, control_result)
-        status_match = sm(experiment_code, control_code)
-
-        if status_match and content_match:
-            log.info("The %s test was unable to detect a captive portal."
-                     % test_name)
-            return True
-        elif status_match and not content_match:
-            log.info("The %s test detected mismatched content, retrying with " \
-                         "fuzzy match enabled." % test_name)
-            content_fuzzy_match, experiment_code = cm(experiment_url, 
-                                                      control_result,
-                                                      fuzzy=True)
-            if content_fuzzy_match:
-                return True
+        if test_name == "user defined":
+            log.info("Running the %s test for %s..." % (test_name, experiment_url))
+            content_match, experiment_code = cm(experiment_url, control_result)
+            status_match = sm(experiment_code, control_code)
+            if status_match and content_match:
+                log.info("The %s test was unable to detect a captive portal." 
+                         % test_name)
+                return True, test_name
+            elif status_match and not content_match:
+                log.info("The %s test detected mismatched content, retrying "
+                         "with fuzzy match enabled." % test_name)
+                content_fuzzy_match, experiment_code = cm(experiment_url, 
+                                                          control_result,
+                                                          fuzzy=True)
+                if content_fuzzy_match:
+                    return True, test_name
+                else:
+                    return False, test_name
             else:
-                return False
+                log.info("The %s test shows that your network is filtered." 
+                         % test_name)
+                return False, test_name
+        
         else:
-            log.info("The %s test shows that your network is filtered, possibly " \
-                         "due to a captive portal." % test_name)
-            return False
-
-        return False
+            log.warn("Ooni is trying to run an undefined captive portal test.")
+            return False, test_name
+        
 
 def run(ooni):
     """
-    Run the CaptivePortal(Test).
+    Runs the CaptivePortal(Test).
+
+    If do_captive_portal_vendor_tests is set to true, then vendor
+    specific captive portal tests will be run.
+
+    If captive_portal = filename.txt, then user-specified tests
+    will be run.
+
+    Either vendor tests or user-defined tests can be run, or both.
     """
     config = ooni.config
     log = ooni.logger
-    assets = [CaptivePortalAsset(os.path.join(config.main.assetdir, 
-                                              config.tests.captive_portal))]
 
+    assets = []
+    if (os.path.isfile(os.path.join(config.main.assetdir,
+                                    config.tests.captive_portal))):
+        assets.append(CaptivePortalAsset(os.path.join(config.main.assetdir, 
+                                                      config.tests.captive_portal)))
+    
     captiveportal = CaptivePortal(ooni)
     log.info("Starting captive portal test...")
+    log.info("Running user defined tests...")
     captiveportal.run(assets, {'index': 1})
-    log.info("Captive portal test finished!")
-
+    
+    if config.tests.do_captive_portal_vendor_tests:
+        log.info("Running vendor tests...")
+        captiveportal.run_vendor_tests()
 
+    log.info("Captive portal test finished!")



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits