[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Iterate on HTTP Invalid Request Line test



commit 1de07f659f1393d969a1b3766baffeecb111355d
Author: Arturo Filastò <art@xxxxxxxxx>
Date:   Thu Nov 29 15:41:01 2012 +0100

    Iterate on HTTP Invalid Request Line test
    * Rename it to HTTP Invalid Request Line
    * Extend it's fuzzing capabilities to support some more specific tests
    * Improve TCPT test template
---
 nettests/blocking/http_invalid_requests.py         |   63 ------------
 nettests/manipulation/http_invalid_request_line.py |  106 ++++++++++++++++++++
 ooni/nettest.py                                    |   38 +++++++-
 ooni/templates/httpt.py                            |   21 +----
 ooni/templates/tcpt.py                             |   48 ++++++----
 5 files changed, 174 insertions(+), 102 deletions(-)

diff --git a/nettests/blocking/http_invalid_requests.py b/nettests/blocking/http_invalid_requests.py
deleted file mode 100644
index 7e6f47f..0000000
--- a/nettests/blocking/http_invalid_requests.py
+++ /dev/null
@@ -1,63 +0,0 @@
-# -*- encoding: utf-8 -*-
-from twisted.python import usage
-
-from ooni.utils import randomStr
-from ooni.templates import tcpt
-
-class UsageOptions(usage.Options):
-    optParameters = [['backend', 'b', '127.0.0.1:57002',
-                        'The OONI backend that runs a TCP echo server (must be on port 80)']]
-
-    optFlags = [['nopayloadmatch', 'n',
-        "Don't match the payload of the response. This option is used when you don't have a TCP echo server running"]]
-
-class HTTPInvalidRequests(tcpt.TCPTest):
-    name = "HTTP Invalid Requests"
-    version = "0.1.1"
-    authors = "Arturo Filastò"
-
-    inputFile = ['file', 'f', None,
-                 'Input file of list of hostnames to attempt to resolve']
-
-    usageOptions = UsageOptions
-    requiredOptions = ['backend']
-
-    def setUp(self):
-        try:
-            self.address, self.port = self.localOptions['backend'].split(":")
-            self.port = int(self.port)
-        except:
-            raise usage.UsageError("Invalid backend address specified (must be address:port)")
-
-    def test_random_invalid_request(self):
-        """
-        We test sending data to a TCP echo server, if what we get back is not
-        what we have sent then there is tampering going on.
-        This is for example what squid will return when performing such
-        request:
-
-            HTTP/1.0 400 Bad Request
-            Server: squid/2.6.STABLE21
-            Date: Sat, 23 Jul 2011 02:22:44 GMT
-            Content-Type: text/html
-            Content-Length: 1178
-            Expires: Sat, 23 Jul 2011 02:22:44 GMT
-            X-Squid-Error: ERR_INVALID_REQ 0
-            X-Cache: MISS from cache_server
-            X-Cache-Lookup: NONE from cache_server:3128
-            Via: 1.0 cache_server:3128 (squid/2.6.STABLE21)
-            Proxy-Connection: close
-
-        """
-        payload = randomStr(10) + "\n\r"
-        def got_all_data(received_array):
-            if not self.localOptions['nopayloadmatch']:
-                first = received_array[0]
-                if first != payload:
-                    self.report['tampering'] = True
-            else:
-                self.report['tampering'] = 'unknown'
-
-        d = self.sendPayload(payload)
-        d.addCallback(got_all_data)
-        return d
diff --git a/nettests/manipulation/http_invalid_request_line.py b/nettests/manipulation/http_invalid_request_line.py
new file mode 100644
index 0000000..00ab24d
--- /dev/null
+++ b/nettests/manipulation/http_invalid_request_line.py
@@ -0,0 +1,106 @@
+# -*- encoding: utf-8 -*-
+from twisted.python import usage
+
+from ooni.utils import randomStr
+from ooni.templates import tcpt
+
+class UsageOptions(usage.Options):
+    optParameters = [['backend', 'b', '127.0.0.1',
+                        'The OONI backend that runs a TCP echo server']]
+
+class HTTPInvalidRequestLine(tcpt.TCPTest):
+    """
+    The goal of this test is to do some very basic and not very noisy fuzzing
+    on the HTTP request line. We generate a series of requests that are not
+    valid HTTP requests.
+
+    Unless elsewhere stated 'Xx'*N refers to N*2 random upper or lowercase ascii
+    letters or numbers ('XxXx' will be 4).
+    """
+    name = "HTTP Invalid Requests"
+    version = "0.1.3"
+    authors = "Arturo Filastò"
+
+    inputFile = ['file', 'f', None,
+                 'Input file of list of hostnames to attempt to resolve']
+
+    usageOptions = UsageOptions
+    requiredOptions = ['backend']
+
+    def setUp(self):
+        self.port = 80
+        self.address = self.localOptions['backend']
+
+    def check_for_manipulation(self, response, payload):
+        if response != payload:
+            self.report['tampering'] = True
+        else:
+            self.report['tampering'] = 'unknown'
+
+    def test_random_invalid_method(self):
+        """
+        We test sending data to a TCP echo server listening on port 80, if what
+        we get back is not what we have sent then there is tampering going on.
+        This is for example what squid will return when performing such
+        request:
+
+            HTTP/1.0 400 Bad Request
+            Server: squid/2.6.STABLE21
+            Date: Sat, 23 Jul 2011 02:22:44 GMT
+            Content-Type: text/html
+            Content-Length: 1178
+            Expires: Sat, 23 Jul 2011 02:22:44 GMT
+            X-Squid-Error: ERR_INVALID_REQ 0
+            X-Cache: MISS from cache_server
+            X-Cache-Lookup: NONE from cache_server:3128
+            Via: 1.0 cache_server:3128 (squid/2.6.STABLE21)
+            Proxy-Connection: close
+
+        """
+        payload = randomStr(10) + " / HTTP/1.1\n\r"
+
+        d = self.sendPayload(payload)
+        d.addCallback(self.check_for_manipulation, payload)
+        return d
+
+    def test_random_invalid_field_count(self):
+        """
+        This generates a request that looks like this:
+
+        XxXxX XxXxX XxXxX XxXxX
+
+        This may trigger some bugs in the HTTP parsers of transparent HTTP
+        proxies.
+        """
+        payload = ' '.join(randomStr(5) for x in range(4))
+        payload += "\n\r"
+
+        d = self.sendPayload(payload)
+        d.addCallback(self.check_for_manipulation, payload)
+        return d
+
+    def test_random_big_request_method(self):
+        """
+        This generates a request that looks like this:
+
+        Xx*512 / HTTP/1.1
+        """
+        payload = randomStr(1024) + ' / HTTP/1.1\n\r'
+
+        d = self.sendPayload(payload)
+        d.addCallback(self.check_for_manipulation, payload)
+        return d
+
+    def test_random_invalid_version_number(self):
+        """
+        This generates a request that looks like this:
+
+        GET / HTTP/XxX
+        """
+        payload = 'GET / HTTP/' + randomStr(3)
+        payload += '\n\r'
+
+        d = self.sendPayload(payload)
+        d.addCallback(self.check_for_manipulation, payload)
+        return d
+
diff --git a/ooni/nettest.py b/ooni/nettest.py
index e0393e7..4a54414 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -5,7 +5,6 @@
 # In here is the NetTest API definition. This is how people
 # interested in writing ooniprobe tests will be specifying them
 #
-# :authors: Arturo Filastò, Isis Lovecruft
 # :license: see included LICENSE file
 
 import sys
@@ -17,12 +16,47 @@ from twisted.trial import unittest, itrial, util
 from twisted.internet import defer, utils
 from twisted.python import usage
 
+from twisted.internet.error import ConnectionRefusedError, DNSLookupError, TCPTimedOutError
+
 from ooni.utils import log
 
+def failureToString(failure):
+    """
+    Given a failure instance return a string representing the kind of error
+    that occurred.
+
+    Args:
+
+        failure: a :class:twisted.internet.error instance
+
+    Returns:
+
+        A string representing the HTTP response error message.
+    """
+    if isinstance(failure.value, ConnectionRefusedError):
+        log.err("Connection refused. The backend may be down")
+        string = 'connection_refused_error'
+
+    elif isinstance(failure.value, SOCKSError):
+        log.err("Sock error. The SOCKS proxy may be down")
+        string = 'socks_error'
+
+    elif isinstance(failure.value, DNSLookupError):
+        log.err("DNS lookup failure")
+        string = 'dns_lookup_error'
+
+    elif isinstance(failure.value, TCPTimedOutError):
+        log.err("TCP Timed Out Error")
+        string = 'tcp_timed_out_error'
+
+    elif isinstance(failure.value, ResponseNeverReceived):
+        log.err("Response Never Received")
+        string = 'response_never_received'
+    return string
+
 class NoPostProcessor(Exception):
     pass
 
-
 class NetTestCase(object):
     """
     This is the base of the OONI nettest universe. When you write a nettest
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 804a3e4..01853f3 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -23,6 +23,8 @@ from ooni import config
 from ooni.utils.net import BodyReceiver, StringProducer, userAgents
 
 from ooni.utils.txagentwithsocks import Agent, SOCKSError, TrueHeaders
+from ooni.nettest import failureToString
+
 
 class InvalidSocksProxyOption(Exception):
     pass
@@ -131,25 +133,8 @@ class HTTPTest(NetTestCase):
                 'code': response.code
         }
         if failure:
-            if isinstance(failure.value, ConnectionRefusedError):
-                log.err("Connection refused. The backend may be down")
-                request_response['failure'] = 'connection_refused_error'
-
-            elif isinstance(failure.value, SOCKSError):
-                log.err("Sock error. The SOCKS proxy may be down")
-                request_response['failure'] = 'socks_error'
-
-            elif isinstance(failure.value, DNSLookupError):
-                log.err("DNS lookup failure")
-                request_response['failure'] = 'dns_lookup_error'
-
-            elif isinstance(failure.value, TCPTimedOutError):
-                log.err("TCP Timed Out Error")
-                request_response['failure'] = 'tcp_timed_out_error'
+            request_response['failure'] = failureToString(failure)
 
-            elif isinstance(failure.value, ResponseNeverReceived):
-                log.err("Response Never Received")
-                request_response['failure'] = 'response_never_received'
         self.report['requests'].append(request_response)
 
     def _processResponseBody(self, response_body, request, response, body_processor):
diff --git a/ooni/templates/tcpt.py b/ooni/templates/tcpt.py
index 77ffe3e..26f38ed 100644
--- a/ooni/templates/tcpt.py
+++ b/ooni/templates/tcpt.py
@@ -2,13 +2,14 @@ from twisted.internet import protocol, defer, reactor
 from twisted.internet.error import ConnectionDone
 from twisted.internet.endpoints import TCP4ClientEndpoint
 
-from ooni.nettest import NetTestCase
+from ooni.nettest import NetTestCase, failureToString
 from ooni.utils import log
 
 class TCPSender(protocol.Protocol):
-    report = None
-    payload_len = None
-    received_data = ''
+    def __init__(self):
+        self.received_data = ''
+        self.sent_data = ''
+
     def dataReceived(self, data):
         """
         We receive data until the total amount of data received reaches that
@@ -27,18 +28,19 @@ class TCPSender(protocol.Protocol):
         """
         if self.payload_len:
             self.received_data += data
-            if len(self.received_data) >= self.payload_len:
-                self.transport.loseConnection()
-                self.report['received'].append(data)
-                self.deferred.callback(self.report['received'])
 
     def sendPayload(self, payload):
         """
         Write the payload to the wire and set the expected size of the payload
         we are to receive.
+
+        Args:
+
+            payload: the data to be sent on the wire.
+
         """
         self.payload_len = len(payload)
-        self.report['sent'].append(payload)
+        self.sent_data = payload
         self.transport.write(payload)
 
 class TCPSenderFactory(protocol.Factory):
@@ -50,7 +52,7 @@ class TCPTest(NetTestCase):
     version = "0.1"
 
     requiresRoot = False
-    timeout = 2
+    timeout = 5
     address = None
     port = None
 
@@ -61,26 +63,34 @@ class TCPTest(NetTestCase):
     def sendPayload(self, payload):
         d1 = defer.Deferred()
 
-        def closeConnection(p):
-            p.transport.loseConnection()
+        def closeConnection(proto):
+            self.report['sent'].append(proto.sent_data)
+            self.report['received'].append(proto.received_data)
+            proto.transport.loseConnection()
             log.debug("Closing connection")
             d1.callback(self.report['received'])
 
+        def timedOut(proto):
+            self.report['failure'] = 'tcp_timed_out_error'
+            proto.transport.loseConnection()
+
         def errback(failure):
-            self.report['error'] = str(failure)
+            self.report['failure'] = failureToString(failure)
             d1.errback(failure)
 
-        def connected(p):
+        def connected(proto):
             log.debug("Connected to %s:%s" % (self.address, self.port))
-            p.report = self.report
-            p.deferred = d1
-            p.sendPayload(payload)
-            reactor.callLater(self.timeout, closeConnection, p)
+            proto.report = self.report
+            proto.deferred = d1
+            proto.sendPayload(payload)
+            if self.timeout:
+                # XXX-Twisted this logic should probably go inside of the protocol
+                reactor.callLater(self.timeout, closeConnection, proto)
 
         point = TCP4ClientEndpoint(reactor, self.address, self.port)
+        log.debug("Connecting to %s:%s" % (self.address, self.port))
         d2 = point.connect(TCPSenderFactory())
         d2.addCallback(connected)
         d2.addErrback(errback)
         return d1
 
-

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits