[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Test and Implement HTTP Header Field Manipulation Test



commit fb5ab197c4efabd005286c32335010f24a102267
Author: Arturo Filastò <art@xxxxxxxxx>
Date:   Sun Nov 25 06:50:37 2012 +0100

    Test and Implement HTTP Header Field Manipulation Test
    (rename it to what we had originally called it since it made most sense)
    * Extend TrueHeaders to support calculation of difference between two HTTP
      headers respectful of capitalization
    * Write unittests for TrueHeaders functions with decent code path coverage
    * Add commented out testdeck for running HTTP Header Field manipulation test
    * Fix bug in calculation of runtime of test
---
 before_i_commit.testdeck                        |    9 +
 nettests/core/http_header_field_manipulation.py |  181 ++++++++++++++++++++++
 nettests/core/http_requests.py                  |  187 -----------------------
 ooni/reporter.py                                |    2 +-
 ooni/utils/txagentwithsocks.py                  |   36 +++++
 oonib/testhelpers/http_helpers.py               |    5 +-
 tests/test_trueheaders.py                       |   41 +++++
 7 files changed, 272 insertions(+), 189 deletions(-)

diff --git a/before_i_commit.testdeck b/before_i_commit.testdeck
index e0b30ea..7fb33f9 100644
--- a/before_i_commit.testdeck
+++ b/before_i_commit.testdeck
@@ -38,3 +38,12 @@
     reportfile: http_url_lists.yamloo
     subargs: [-f, test_inputs/url_lists_file.txt]
     test: nettests/core/http_url_list.py
+# XXX this is disabled because it requires oonib to be running
+#- options:
+#    collector: null
+#    help: 0
+#    logfile: null
+#    pcapfile: null
+#    reportfile: null
+#    subargs: [-h, test_inputs/test_header_field_manipulation.txt]
+#    test: nettests/core/http_header_field_manipulation.py
diff --git a/nettests/core/http_header_field_manipulation.py b/nettests/core/http_header_field_manipulation.py
new file mode 100644
index 0000000..08ee8c7
--- /dev/null
+++ b/nettests/core/http_header_field_manipulation.py
@@ -0,0 +1,181 @@
+# -*- encoding: utf-8 -*-
+#
+# :authors: Arturo Filastò
+# :licence: see LICENSE
+
+import random
+import json
+import yaml
+
+from twisted.python import usage
+
+from ooni.utils import log, net, randomStr
+from ooni.templates import httpt
+from ooni.utils.txagentwithsocks import TrueHeaders
+
+def random_capitalization(string):
+    output = ""
+    original_string = string
+    string = string.swapcase()
+    for i in range(len(string)):
+        if random.randint(0, 1):
+            output += string[i].swapcase()
+        else:
+            output += string[i]
+    if original_string == output:
+        return random_capitalization(output)
+    else:
+        return output
+
+class UsageOptions(usage.Options):
+    optParameters = [
+            ['backend', 'b', 'http://127.0.0.1:57001', 
+                'URL of the backend to use for sending the requests'],
+            ['headers', 'h', None,
+                'Specify a yaml formatted file from which to read the request headers to send']
+            ]
+
+class HTTPHeaderFieldManipulation(httpt.HTTPTest):
+    """
+    It performes HTTP requests with request headers that vary capitalization
+    towards a backend. If we detect that the headers the backend received
+    matches the ones we have sent then we have detected tampering.
+    """
+    name = "HTTP Header Field Manipulation"
+    author = "Arturo Filastò"
+    version = "0.1.3"
+
+    randomizeUA = False
+    usageOptions = UsageOptions
+
+    requiredOptions = ['backend']
+
+    def get_headers(self):
+        headers = {}
+        if self.localOptions['headers']:
+            try:
+                f = open(self.localOptions['headers'])
+            except IOError:
+                raise Exception("Specified input file does not exist")
+            content = ''.join(f.readlines())
+            f.close()
+            headers = yaml.safe_load(content)
+            return headers
+        else:
+            # XXX generate these from a random choice taken from whatheaders.com
+            # http://s3.amazonaws.com/data.whatheaders.com/whatheaders-latest.xml.zip
+            headers = {"User-Agent": [random.choice(net.userAgents)[0]],
+                "Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
+                "Accept-Encoding": ["gzip,deflate,sdch"],
+                "Accept-Language": ["en-US,en;q=0.8"],
+                "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"],
+                "Host": [randomStr(15)+'.com']
+            }
+            return headers
+
+    def get_random_caps_headers(self):
+        headers = {}
+        normal_headers = self.get_headers()
+        for k, v in normal_headers.items():
+            new_key = random_capitalization(k)
+            headers[new_key] = v
+        return headers
+
+    def processInputs(self):
+        if self.localOptions['backend']:
+            self.url = self.localOptions['backend']
+        else:
+            raise Exception("No backend specified")
+
+    def processResponseBody(self, data):
+        self.check_for_tampering(data)
+
+    def check_for_tampering(self, data):
+        """
+        Here we do checks to verify if the request we made has been tampered
+        with. We have 3 categories of tampering:
+
+        *  **total** when the response is not a json object and therefore we were not
+        able to reach the ooniprobe test backend
+
+        *  **request_line_capitalization** when the HTTP Request line (e.x. GET /
+        HTTP/1.1) does not match the capitalization we set.
+
+        *  **header_field_number** when the number of headers we sent does not match
+        with the ones the backend received
+
+        *  **header_name_capitalization** when the header field names do not match
+        those that we sent.
+
+        *  **header_field_value** when the header field value does not match with the
+        one we transmitted.
+        """
+        self.report['tampering'] = {
+            'total': False,
+            'request_line_capitalization': False,
+            'header_name_capitalization': False,
+            'header_field_value': False,
+            'header_field_number': False
+        }
+        try:
+            response = json.loads(data)
+        except ValueError:
+            self.report['tampering']['total'] = True
+            return
+
+        request_request_line = "%s / HTTP/1.1" % self.request_method
+
+        try:
+            response_request_line = response['request_line']
+            response_headers_dict = response['headers_dict']
+        except KeyError:
+            self.report['tampering']['total'] = True
+            return
+
+        if request_request_line != response_request_line:
+            self.report['tampering']['request_line_capitalization'] = True
+
+        request_headers = TrueHeaders(self.request_headers)
+        diff = request_headers.getDiff(response_headers_dict, ignore=['Connection'])
+        if diff:
+            self.report['tampering']['header_field_name'] = True
+        else:
+            self.report['tampering']['header_field_name'] = False
+        self.report['tampering']['header_name_diff'] = list(diff)
+
+    def test_get(self):
+        self.request_method = "GET"
+        self.request_headers = self.get_random_caps_headers()
+        return self.doRequest(self.url, self.request_method,
+                headers=self.request_headers)
+
+    def test_get_random_capitalization(self):
+        self.request_method = random_capitalization("GET")
+        self.request_headers = self.get_random_caps_headers()
+        return self.doRequest(self.url, self.request_method,
+                headers=self.request_headers)
+
+    def test_post(self):
+        self.request_method = "POST"
+        self.request_headers = self.get_headers()
+        return self.doRequest(self.url, self.request_method,
+                headers=self.request_headers)
+
+    def test_post_random_capitalization(self):
+        self.request_method = random_capitalization("POST")
+        self.request_headers = self.get_random_caps_headers()
+        return self.doRequest(self.url, self.request_method,
+                headers=self.request_headers)
+
+    def test_put(self):
+        self.request_method = "PUT"
+        self.request_headers = self.get_headers()
+        return self.doRequest(self.url, self.request_method,
+                headers=self.request_headers)
+
+    def test_put_random_capitalization(self):
+        self.request_method = random_capitalization("PUT")
+        self.request_headers = self.get_random_caps_headers()
+        return self.doRequest(self.url, self.request_method,
+                headers=self.request_headers)
+
diff --git a/nettests/core/http_requests.py b/nettests/core/http_requests.py
deleted file mode 100644
index 5d67070..0000000
--- a/nettests/core/http_requests.py
+++ /dev/null
@@ -1,187 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# :authors: Arturo Filastò
-# :licence: see LICENSE
-
-import random
-import json
-
-from twisted.python import usage
-
-from ooni.utils import log, net, randomStr
-from ooni.templates import httpt
-
-def random_capitalization(string):
-    output = ""
-    original_string = string
-    string = string.swapcase()
-    for i in range(len(string)):
-        if random.randint(0, 1):
-            output += string[i].swapcase()
-        else:
-            output += string[i]
-    if original_string == output:
-        return random_capitalization(output)
-    else:
-        return output
-
-class UsageOptions(usage.Options):
-    optParameters = [
-            ['backend', 'b', 'http://127.0.0.1:57001', 
-                'URL of the backend to use for sending the requests'],
-            ['headers', 'h', None,
-                'Specify a yaml formatted file from which to read the request headers to send']
-            ]
-
-class HTTPRequests(httpt.HTTPTest):
-    """
-    This test is also known as Header Field manipulation. It performes HTTP
-    requests with variations in capitalization towards the backend.
-    """
-    name = "HTTP Requests"
-    author = "Arturo Filastò"
-    version = "0.1.1"
-
-    randomizeUA = False
-    usageOptions = UsageOptions
-
-    requiredOptions = ['backend']
-
-    def processInputs(self):
-        if self.localOptions['backend']:
-            self.url = self.localOptions['backend']
-        else:
-            raise Exception("No backend specified")
-
-    def processResponseBody(self, data):
-        self.check_for_tampering(data)
-
-    def check_for_tampering(self, data):
-        """
-        Here we do checks to verify if the request we made has been tampered
-        with. We have 3 categories of tampering:
-
-        *  **total** when the response is not a json object and therefore we were not
-        able to reach the ooniprobe test backend
-
-        *  **request_line_capitalization** when the HTTP Request line (e.x. GET /
-        HTTP/1.1) does not match the capitalization we set.
-
-        *  **header_field_number** when the number of headers we sent does not match
-        with the ones the backend received
-
-        *  **header_name_capitalization** when the header field names do not match
-        those that we sent.
-
-        *  **header_field_value** when the header field value does not match with the
-        one we transmitted.
-        """
-        self.report['tampering'] = {'total': False,
-                'request_line_capitalization': False, 
-                'header_name_capitalization': False,
-                'header_field_value': False,
-                'header_field_number': False
-        }
-
-        try:
-            response = json.loads(data)
-        except ValueError:
-            self.report['tampering']['total'] = True
-            return
-
-        requestLine = "%s / HTTP/1.1" % self.request_method
-        if response['request_line'] != requestLine:
-            self.report['tampering']['request_line_capitalization'] = True
-
-        # We compare against length -1 because the response will also contain
-        # the Connection: close header since we do not do persistent
-        # connections
-        if len(self.request_headers) != (len(response['headers_dict']) - 1):
-            self.report['tampering']['header_field_number'] = True
-
-        for header, value in self.request_headers.items():
-            # XXX this still needs some work
-            # in particular if the response headers are of different length or
-            # some extra headers get added in the response (so the lengths
-            # match), we will get header_name_capitalization set to true, while
-            # the actual tampering is the addition of an extraneous header
-            # field.
-            if header == "Connection":
-                # Ignore Connection header
-                continue
-            try:
-                response_value = response['headers_dict'][header]
-                if response_value != value[0]:
-                    log.msg("Tampering detected because %s != %s" % (response_value, value[0]))
-                    self.report['tampering']['header_field_value'] = True
-            except KeyError:
-                log.msg("Tampering detected because %s not in %s" % (header, response['headers_dict']))
-                self.report['tampering']['header_name_capitalization'] = True
-
-    def get_headers(self):
-        headers = {}
-        if self.localOptions['headers']:
-            # XXX test this code
-            try:
-                f = open(self.localOptions['headers'])
-            except IOError:
-                raise Exception("Specified input file does not exist")
-            content = ''.join(f.readlines())
-            f.close()
-            headers = yaml.load(content)
-            return headers
-        else:
-            headers = {"User-Agent": [random.choice(net.userAgents)[0]],
-                "Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
-                "Accept-Encoding": ["gzip,deflate,sdch"],
-                "Accept-Language": ["en-US,en;q=0.8"],
-                "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"],
-                "Host": [randomStr(15)+'.com']
-            }
-            return headers
-
-    def get_random_caps_headers(self):
-        headers = {}
-        normal_headers = self.get_headers()
-        for k, v in normal_headers.items():
-            new_key = random_capitalization(k)
-            headers[new_key] = v
-        return headers
-
-    def test_get(self):
-        self.request_method = "GET"
-        self.request_headers = self.get_random_caps_headers()
-        return self.doRequest(self.url, self.request_method,
-                headers=self.request_headers)
-
-    def test_get_random_capitalization(self):
-        self.request_method = random_capitalization("GET")
-        self.request_headers = self.get_random_caps_headers()
-        return self.doRequest(self.url, self.request_method,
-                headers=self.request_headers)
-
-    def test_post(self):
-        self.request_method = "POST"
-        self.request_headers = self.get_headers()
-        return self.doRequest(self.url, self.request_method,
-                headers=self.request_headers)
-
-    def test_post_random_capitalization(self):
-        self.request_method = random_capitalization("POST")
-        self.request_headers = self.get_random_caps_headers()
-        return self.doRequest(self.url, self.request_method,
-                headers=self.request_headers)
-
-    def test_put(self):
-        self.request_method = "PUT"
-        self.request_headers = self.get_headers()
-        return self.doRequest(self.url, self.request_method,
-                headers=self.request_headers)
-
-    def test_put_random_capitalization(self):
-        self.request_method = random_capitalization("PUT")
-        self.request_headers = self.get_random_caps_headers()
-        return self.doRequest(self.url, self.request_method,
-                headers=self.request_headers)
-
-
diff --git a/ooni/reporter.py b/ooni/reporter.py
index b79f27f..63f501e 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -184,7 +184,7 @@ class OReporter(object):
             test_input = test.input
 
         test_started = test._start_time
-        test_runtime = test_started - time.time()
+        test_runtime = time.time() - test_started
 
         report = {'input': test_input,
                 'test_name': test_name,
diff --git a/ooni/utils/txagentwithsocks.py b/ooni/utils/txagentwithsocks.py
index 57c27e4..7b000fc 100644
--- a/ooni/utils/txagentwithsocks.py
+++ b/ooni/utils/txagentwithsocks.py
@@ -4,6 +4,8 @@
 # :licence: see LICENSE
 
 import struct
+import itertools
+from copy import copy
 
 from zope.interface import implements
 from twisted.web import client, _newclient, http_headers
@@ -137,6 +139,40 @@ class TrueHeaders(http_headers.Headers):
         self._rawHeaders[name.lower()]['name'] = name
         self._rawHeaders[name.lower()]['values'] = values
 
+    def getDiff(self, header_dict, ignore=[]):
+        """
+        ignore: specify a list of header fields to ignore
+
+        Returns a set containing the header names that are not present in
+        header_dict or not present in self.
+        """
+        diff = set()
+        field_names = []
+
+        headers_a = copy(self)
+        headers_b = TrueHeaders(header_dict)
+        for name in ignore:
+            try:
+                del headers_a._rawHeaders[name.lower()]
+            except KeyError:
+                pass
+            try:
+                del headers_b._rawHeaders[name.lower()]
+            except KeyError:
+                pass
+
+        for k, v in itertools.chain(headers_a.getAllRawHeaders(), \
+                headers_b.getAllRawHeaders()):
+            field_names.append(k)
+
+        for name in field_names:
+            if self.getRawHeaders(name) and \
+                name in header_dict:
+                pass
+            else:
+                diff.add(name)
+        return diff
+
     def getAllRawHeaders(self):
         for k, v in self._rawHeaders.iteritems():
             yield v['name'], v['values']
diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py
index b384216..1fa0ccb 100644
--- a/oonib/testhelpers/http_helpers.py
+++ b/oonib/testhelpers/http_helpers.py
@@ -77,7 +77,10 @@ class SimpleHTTPChannel(basic.LineReceiver, policies.TimeoutMixin):
     def allHeadersReceived(self):
         headers_dict = {}
         for k, v in self.headers:
-            headers_dict[k] = v
+            if k not in headers_dict:
+                headers_dict[k] = []
+            headers_dict[k].append(v)
+
         response = {'request_headers': self.headers,
             'request_line': self.requestLine,
             'headers_dict': headers_dict
diff --git a/tests/test_trueheaders.py b/tests/test_trueheaders.py
new file mode 100644
index 0000000..33521b8
--- /dev/null
+++ b/tests/test_trueheaders.py
@@ -0,0 +1,41 @@
+from twisted.trial import unittest
+
+from ooni.utils.txagentwithsocks import TrueHeaders
+
+dummy_headers_dict = {
+        'Header1': ['Value1', 'Value2'],
+        'Header2': ['ValueA', 'ValueB']
+}
+
+dummy_headers_dict2 = {
+        'Header1': ['Value1', 'Value2'],
+        'Header2': ['ValueA', 'ValueB'],
+        'Header3': ['ValueA', 'ValueB'],
+}
+
+dummy_headers_dict3 = {
+        'Header1': ['Value1', 'Value2'],
+        'Header2': ['ValueA', 'ValueB'],
+        'Header4': ['ValueA', 'ValueB'],
+}
+
+
+class TestTrueHeaders(unittest.TestCase):
+    def test_names_match(self):
+        th = TrueHeaders(dummy_headers_dict)
+        self.assertEqual(th.getDiff(dummy_headers_dict), set())
+
+    def test_names_not_match(self):
+        th = TrueHeaders(dummy_headers_dict)
+        self.assertEqual(th.getDiff(dummy_headers_dict2), set(['Header3']))
+
+        th = TrueHeaders(dummy_headers_dict3)
+        self.assertEqual(th.getDiff(dummy_headers_dict2), set(['Header3', 'Header4']))
+
+    def test_names_match_expect_ignore(self):
+        th = TrueHeaders(dummy_headers_dict)
+        self.assertEqual(th.getDiff(dummy_headers_dict2, ignore=['Header3']), set())
+
+
+
+

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits