[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Port squid transparent HTTP proxy detector to new API
commit f5acc677c00b8ab83bc6dc41fdd09ac487acdfc0
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date: Tue Oct 23 18:04:36 2012 +0000
Port squid transparent HTTP proxy detector to new API
* Remove some dead code
* Move authors to new directory
* Make reporter not swallow all tracebacks
* Fix some bugs in httpt
---
AUTHORS | 4 +
nettests/core/captiveportal.py | 19 +++
nettests/core/squid.py | 117 ++++++++++++++++++++
old-to-be-ported-code/AUTHORS | 3 -
old-to-be-ported-code/bin/ooni-probe | 10 --
old-to-be-ported-code/ooni/output.py | 21 ----
.../ooni/plugins/captiveportal_plgoo.py | 55 ---------
old-to-be-ported-code/ooni/plugins/skel_plgoo.py | 17 ---
old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml | 33 ------
old-to-be-ported-code/ooni/yamlooni.py | 40 -------
ooni/nettest.py | 97 +++++++----------
ooni/reporter.py | 23 +++-
ooni/runner.py | 11 ++-
ooni/templates/httpt.py | 47 +++++++-
ooni/utils/__init__.py | 32 ++++++
15 files changed, 276 insertions(+), 253 deletions(-)
diff --git a/AUTHORS b/AUTHORS
new file mode 100644
index 0000000..d8dc0b8
--- /dev/null
+++ b/AUTHORS
@@ -0,0 +1,4 @@
+Jacob Appelbaum <jacob@xxxxxxxxxxxxxx>
+Arturo Filasto <hellais@xxxxxxxxxxxxxx>
+Linus Nordberg <linus@xxxxxxxxxxxxxx>
+Isis Lovecruft <isis@xxxxxxxxxxxxxx>
diff --git a/nettests/core/captiveportal.py b/nettests/core/captiveportal.py
index 46f0856..77ba3e4 100644
--- a/nettests/core/captiveportal.py
+++ b/nettests/core/captiveportal.py
@@ -7,6 +7,25 @@
captive portal. Code is taken, in part, from the old ooni-probe,
which was written by Jacob Appelbaum and Arturo Filastò.
+ This module performs multiple tests that match specific vendor captive
+ portal tests. This is a basic internet captive portal filter tester written
+ for RECon 2011.
+
+ Read the following URLs to understand the captive portal detection process
+ for various vendors:
+
+ http://technet.microsoft.com/en-us/library/cc766017%28WS.10%29.aspx
+ http://blog.superuser.com/2011/05/16/windows-7-network-awareness/
+ http://isc.sans.org/diary.html?storyid=10312&;
+ http://src.chromium.org/viewvc/chrome?view=rev&revision=74608
+ http://code.google.com/p/chromium-os/issues/detail?3281ttp,
+ http://crbug.com/52489
+ http://crbug.com/71736
+ https://bugzilla.mozilla.org/show_bug.cgi?id=562917
+ https://bugzilla.mozilla.org/show_bug.cgi?id=603505
+ http://lists.w3.org/Archives/Public/ietf-http-wg/2011JanMar/0086.html
+ http://tools.ietf.org/html/draft-nottingham-http-portal-02
+
:copyright: (c) 2012 Isis Lovecruft
:license: see LICENSE for more details
"""
diff --git a/nettests/core/squid.py b/nettests/core/squid.py
new file mode 100644
index 0000000..675119c
--- /dev/null
+++ b/nettests/core/squid.py
@@ -0,0 +1,117 @@
+# -*- encoding: utf-8 -*-
+#
+# Squid transparent HTTP proxy detector
+# *************************************
+#
+# :authors: Arturo Filastò
+# :licence: see LICENSE
+
+from ooni import utils
+from ooni.utils import log
+from ooni.templates import httpt
+
+class SquidTest(httpt.HTTPTest):
+ """
+ This test aims at detecting the presence of a squid based transparent HTTP
+ proxy. It also tries to detect the version number.
+ """
+ name = "Squid test"
+ author = "Arturo Filastò"
+ version = 0.1
+
+ optParameters = [['backend', 'b', 'http://ooni.nu/test/', 'Test backend to use']]
+
+ #inputFile = ['urls', 'f', None, 'Urls file']
+ inputs =['http://google.com']
+ def test_cacheobject(self):
+ """
+ This detects the presence of a squid transparent HTTP proxy by sending
+ a request for cache_object://localhost/info.
+
+ The response to this request will usually also contain the squid
+ version number.
+ """
+ log.debug("Running")
+ def process_body(body):
+ if "Access Denied." in body:
+ self.report['transparent_http_proxy'] = True
+ else:
+ self.report['transparent_http_proxy'] = False
+
+ log.msg("Testing Squid proxy presence by sending a request for "\
+ "cache_object")
+ headers = {}
+ #headers["Host"] = [self.input]
+ self.report['trans_http_proxy'] = None
+ method = "GET"
+ body = "cache_object://localhost/info"
+ return self.doRequest(self.localOptions['backend'], method=method, body=body,
+ headers=headers, body_processor=process_body)
+
+ def test_search_bad_request(self):
+ """
+ Attempts to perform a request with a random invalid HTTP method.
+
+ If we are being MITMed by a Transparent Squid HTTP proxy we will get
+ back a response containing the X-Squid-Error header.
+ """
+ def process_headers(headers):
+ log.debug("Processing headers in test_search_bad_request")
+ if 'X-Squid-Error' in headers:
+ log.msg("Detected the presence of a transparent HTTP "\
+ "squid proxy")
+ self.report['trans_http_proxy'] = True
+ else:
+ log.msg("Did not detect the presence of transparent HTTP "\
+ "squid proxy")
+ self.report['transparent_http_proxy'] = False
+
+ log.msg("Testing Squid proxy presence by sending a random bad request")
+ headers = {}
+ #headers["Host"] = [self.input]
+ method = utils.randomSTR(10, True)
+ self.report['transparent_http_proxy'] = None
+ return self.doRequest(self.localOptions['backend'], method=method,
+ headers=headers, headers_processor=process_headers)
+
+ def test_squid_headers(self):
+ """
+ Detects the presence of a squid transparent HTTP proxy based on the
+ response headers it adds to the responses to requests.
+ """
+ def process_headers(headers):
+ """
+ Checks if any of the headers that squid is known to add match the
+ squid regexp.
+
+ We are looking for something that looks like this:
+
+ via: 1.0 cache_server:3128 (squid/2.6.STABLE21)
+ x-cache: MISS from cache_server
+ x-cache-lookup: MISS from cache_server:3128
+ """
+ squid_headers = {'via': r'.* \((squid.*)\)',
+ 'x-cache': r'MISS from (\w+)',
+ 'x-cache-lookup': r'MISS from (\w+:?\d+?)'
+ }
+
+ self.report['transparent_http_proxy'] = False
+ for key in squid_headers.keys():
+ if key in headers:
+ log.debug("Found %s in headers" % key)
+ m = re.search(squid_headers[key], headers[key])
+ if m:
+ log.msg("Detected the presence of squid transparent"\
+ " HTTP Proxy")
+ self.report['transparent_http_proxy'] = True
+
+ log.msg("Testing Squid proxy by looking at response headers")
+ headers = {}
+ #headers["Host"] = [self.input]
+ method = "GET"
+ self.report['transparent_http_proxy'] = None
+ d = self.doRequest(self.localOptions['backend'], method=method,
+ headers=headers, headers_processor=process_headers)
+ return d
+
+
diff --git a/old-to-be-ported-code/AUTHORS b/old-to-be-ported-code/AUTHORS
deleted file mode 100644
index c6a4ab6..0000000
--- a/old-to-be-ported-code/AUTHORS
+++ /dev/null
@@ -1,3 +0,0 @@
-Jacob Appelbaum <jacob@xxxxxxxxxxxxxx>
-Arturo Filasto <hellais@xxxxxxxxxxxxxx>
-Linus Nordberg <linus@xxxxxxxxxxxxxx>
diff --git a/old-to-be-ported-code/bin/ooni-probe b/old-to-be-ported-code/bin/ooni-probe
deleted file mode 100644
index 9f616bc..0000000
--- a/old-to-be-ported-code/bin/ooni-probe
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env python
-"""\
- This is the example OONI probe command line utility
-"""
-
-import sys
-
-from ooni.command import Command
-
-Command(sys.argv[1:]).run()
diff --git a/old-to-be-ported-code/ooni/output.py b/old-to-be-ported-code/ooni/output.py
deleted file mode 100644
index 48e9f1f..0000000
--- a/old-to-be-ported-code/ooni/output.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import yaml
-
-class data:
- def __init__(self, name=None):
- if name:
- self.name = name
-
- def output(self, data, name=None):
- if name:
- self.name = name
-
- stream = open(self.name, 'w')
- yaml.dump(data, stream)
- stream.close()
- def append(self, data, name=None):
- if name:
- self.name = name
- stream = open(self.name, 'a')
- yaml.dump([data], stream)
- stream.close()
-
diff --git a/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py b/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py
deleted file mode 100644
index 9c0d87c..0000000
--- a/old-to-be-ported-code/ooni/plugins/captiveportal_plgoo.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/env python
-#
-# Captive Portal Detection With Multi-Vendor Emulation
-# by Jacob Appelbaum <jacob@xxxxxxxxxxxxx>
-#
-# This module performs multiple tests that match specific vendor captive
-# portal tests. This is a basic internet captive portal filter tester written
-# for RECon 2011.
-#
-# Read the following URLs to understand the captive portal detection process
-# for various vendors:
-#
-# http://technet.microsoft.com/en-us/library/cc766017%28WS.10%29.aspx
-# http://blog.superuser.com/2011/05/16/windows-7-network-awareness/
-# http://isc.sans.org/diary.html?storyid=10312&;
-# http://src.chromium.org/viewvc/chrome?view=rev&revision=74608
-# http://code.google.com/p/chromium-os/issues/detail?id=3281
-# http://crbug.com/52489
-# http://crbug.com/71736
-# https://bugzilla.mozilla.org/show_bug.cgi?id=562917
-# https://bugzilla.mozilla.org/show_bug.cgi?id=603505
-# http://lists.w3.org/Archives/Public/ietf-http-wg/2011JanMar/0086.html
-# http://tools.ietf.org/html/draft-nottingham-http-portal-02
-#
-
-import sys
-import ooni.http
-import ooni.dnsooni
-import ooni.report
-
-from ooni.plugooni import Plugoo
-
-class CaptivePortalPlugin(Plugoo):
- def __init__(self):
- self.in_ = sys.stdin
- self.out = sys.stdout
- self.debug = False
- self.logger = ooni.report.Log().logger
- self.name = ""
- self.type = ""
- self.paranoia = ""
- self.modules_to_import = []
- self.output_dir = ""
- self.default_args = ""
-
- def CaptivePortal_Tests(self):
- print "Captive Portal Detection With Multi-Vendor Emulation:"
- tests = self.get_tests_by_filter(("_CP_Tests"), (ooni.http, ooni.dnsooni))
- self.run_tests(tests)
-
- def magic_main(self):
- self.run_plgoo_tests("_Tests")
-
- def ooni_main(self,args):
- self.magic_main()
diff --git a/old-to-be-ported-code/ooni/plugins/skel_plgoo.py b/old-to-be-ported-code/ooni/plugins/skel_plgoo.py
deleted file mode 100644
index f365c06..0000000
--- a/old-to-be-ported-code/ooni/plugins/skel_plgoo.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/python
-# This will never load it is just an example of Plugooni plgoo plugins
-#
-from ooni.plugooni import Plugoo
-
-class SkelPlugin(Plugoo):
- def __init__(self):
- self.name = ""
- self.type = ""
- self.paranoia = ""
- self.modules_to_import = []
- self.output_dir = ""
-
- def ooni_main(self, cmd):
- print "This is the main plugin function"
-
-
diff --git a/old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml b/old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml
deleted file mode 100644
index 6a91e8a..0000000
--- a/old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml
+++ /dev/null
@@ -1,33 +0,0 @@
----
-plugin:
- name : Skel
- author : Some Name
- date_created : 2011-08-01
- modules : [tcp, udp, http]
-input:
- experiment:
- list : ['el1',
- 'el2',
- 'el3',
- 'el4',
- 'el5']
- control:
- list : ['el1',
- 'el2',
- 'el3',
- 'el4',
- 'el5']
-output:
- timestamp :
- experiment :
- timestamp :
- test :
- result :
- extrafield :
- control :
- timestamp :
- test :
- result :
- extrafield :
-
-
diff --git a/old-to-be-ported-code/ooni/yamlooni.py b/old-to-be-ported-code/ooni/yamlooni.py
deleted file mode 100644
index a457217..0000000
--- a/old-to-be-ported-code/ooni/yamlooni.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-#
-# Plugooni, ooni plugin module for loading plgoo files.
-# by Jacob Appelbaum <jacob@xxxxxxxxxxxxx>
-# Arturo Filasto' <art@xxxxxxxxx>
-
-import sys
-import os
-
-class Yamlooni():
- def __init__(self, name, creator, location):
- self.name = name
- self.creator = creator
- self.location = location
- f = open(self.location)
- self.ydata = yaml.load(f.read())
-
- def debug_print():
- #print y.input
- for i in y.iteritems():
- if i[0] == "input":
- print "This is the input part:"
- for j in i[1].iteritems():
- print j
- print "end of the input part.\n"
-
- elif i[0] == "output":
- print "This is the output part:"
- for j in i[1].iteritems():
- print j
- print "end of the output part.\n"
-
- elif i[0] == "plugin":
- print "This is the Plugin part:"
- for j in i[1].iteritems():
- print j
- print "end of the plugin part.\n"
-
-
-
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 3c87b7f..b17117d 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -3,19 +3,9 @@
import itertools
import os
-from inspect import classify_class_attrs
-from pprint import pprint
-
-from twisted.internet import defer, utils
-from twisted.python import usage
-from twisted.trial import unittest, itrial
-from zope.interface.exceptions import BrokenImplementation
-
-from ooni.inputunit import InputUnitProcessor
-from ooni.utils import log
-from ooni.utils.assertions import isClass, isNotClass
-from ooni.utils.assertions import isOldStyleClass, isNewStyleClass
-
+from twisted.trial import unittest, itrial, util
+from twisted.internet import defer, utils
+from ooni.utils import log
pyunit = __import__('unittest')
@@ -25,20 +15,46 @@ class InputTestSuite(pyunit.TestSuite):
and the tracking of current index via idx.
"""
def run(self, result, idx=0):
+ log.debug("Running test suite")
self._idx = idx
while self._tests:
if result.shouldStop:
+ log.debug("Detected that test should stop")
+ log.debug("Stopping...")
break
test = self._tests.pop(0)
+
try:
+ log.debug("Setting test attributes with %s %s" %
+ (self.input, self._idx))
+
test.input = self.input
test._idx = self._idx
+ except Exception, e:
+ log.debug("Error in some stuff")
+ log.debug(e)
+ import sys
+ print sys.exc_info()
+
+ try:
+ log.debug("Running test")
test(result)
- except:
+ log.debug("Ran.")
+ except Exception, e:
+ log.debug("Attribute error thing")
+ log.debug("Had some problems with _idx")
+ log.debug(e)
+ import traceback, sys
+ print sys.exc_info()
+ traceback.print_exc()
+ print e
+
test(result)
+
self._idx += 1
return result
+
class TestCase(unittest.TestCase):
"""
This is the monad of the OONI nettest universe. When you write a nettest
@@ -123,60 +139,23 @@ class TestCase(unittest.TestCase):
writing.
"""
if result.reporterFactory.firstrun:
+ log.debug("Detecting first run. Writing report header.")
d1 = result.reporterFactory.writeHeader()
d2 = unittest.TestCase.deferSetUp(self, ignored, result)
dl = defer.DeferredList([d1, d2])
return dl
else:
+ log.debug("Not first run. Running test setup directly")
return unittest.TestCase.deferSetUp(self, ignored, result)
- def _raaun(self, methodName, result):
- from twisted.internet import reactor
- method = getattr(self, methodName)
- log.debug("Running %s" % methodName)
- d = defer.maybeDeferred(
- utils.runWithWarningsSuppressed, self._getSuppress(), method)
- d.addBoth(lambda x : call.active() and call.cancel() or x)
- return d
-
- @staticmethod
- def inputParser(inputs):
- """Replace me with a custom function for parsing inputs."""
- return inputs
-
- def __input_file_processor__(self, fp):
- """
- I open :attr:inputFile if there is one, and return inputs one by one
- after stripping them of whitespace and running them through the parser
- :meth:`inputParser`.
- """
- for line in fp.readlines():
- yield self.inputParser(line.strip())
+ def inputProcessor(self, fp):
+ log.debug("Running default input processor")
+ for x in fp.readlines():
+ yield x.strip()
fp.close()
- def __get_inputs__(self):
- """
- I am called from the ooni.runner and you probably should not override
- me. I gather the internal inputs from an instantiated test class and
- pass them to the rest of the runner.
-
- If you are looking for a way to parse inputs from inputFile, see
- :meth:`inputParser`.
- """
- processor = InputUnitProcessor(self.inputs,
- input_filter=None,
- catch_err=False)
- processed = processor.process()
-
- log.msg("Received direct inputs:\n%s" % inputs)
- log.debug("Our InputUnitProcessor is %s" % processor)
-
- #while processed is not StopIteration:
- # self.inputs = processed
- # yield self.inputs
- #else:
- # if self.inputFile:
-
+ def getOptions(self):
+ log.debug("Getting options for test")
if self.inputFile:
try:
fp = open(self.inputFile) ## xxx fixme:
diff --git a/ooni/reporter.py b/ooni/reporter.py
index a7b645b..c12b28f 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -4,6 +4,7 @@ import logging
import sys
import time
import yaml
+import traceback
from yaml.representer import *
from yaml.emitter import *
@@ -132,8 +133,7 @@ class ReporterFactory(OReporter):
client_geodata = {}
log.msg("Running geo IP lookup via check.torproject.org")
- #client_ip = yield geodata.myIP()
- client_ip = '127.0.0.1'
+ client_ip = yield geodata.myIP()
try:
import txtorcon
client_location = txtorcon.util.NetLocation(client_ip)
@@ -200,6 +200,7 @@ class OONIReporter(OReporter):
if not self._startTime:
self._startTime = self._getTime()
+ log.debug("Starting test %s" % idx)
test.report = {}
self._tests[idx] = {}
@@ -215,6 +216,7 @@ class OONIReporter(OReporter):
def stopTest(self, test):
+ log.debug("Stopping test")
super(OONIReporter, self).stopTest(test)
idx = self.getTestIndex(test)
@@ -224,11 +226,14 @@ class OONIReporter(OReporter):
# XXX In the future this should be removed.
try:
report = list(test.legacy_report)
+ log.debug("Set the report to be a list")
except:
# XXX I put a dict() here so that the object is re-instantiated and I
# actually end up with the report I want. This could either be a
# python bug or a yaml bug.
report = dict(test.report)
+ log.debug("Set the report to be a dict")
+
log.debug("Adding to report %s" % report)
self._tests[idx]['report'] = report
@@ -245,6 +250,7 @@ class OONIReporter(OReporter):
Expects that L{_printErrors}, L{_writeln}, L{_write}, L{_printSummary}
and L{_separator} are all implemented.
"""
+ log.debug("Test run concluded")
if self._publisher is not None:
self._publisher.removeObserver(self._observeWarnings)
if self._startTime is not None:
@@ -261,13 +267,18 @@ class OONIReporter(OReporter):
super(OONIReporter, self).addSuccess(test)
#self.report['result'] = {'value': 'success'}
- def addError(self, *args):
- super(OONIReporter, self).addError(*args)
- #self.report['result'] = {'value': 'error', 'args': args}
+ def addError(self, test, exception):
+ super(OONIReporter, self).addError(test, exception)
+ exc_type, exc_value, exc_traceback = exception
+ log.err(exc_type)
+ log.err(str(exc_value))
+ # XXX properly print out the traceback
+ for line in '\n'.join(traceback.format_tb(exc_traceback)).split("\n"):
+ log.err(line)
def addFailure(self, *args):
super(OONIReporter, self).addFailure(*args)
- #self.report['result'] = {'value': 'failure', 'args': args}
+ log.warn(args)
def addSkip(self, *args):
super(OONIReporter, self).addSkip(*args)
diff --git a/ooni/runner.py b/ooni/runner.py
index b1a21ac..f352dfb 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -235,13 +235,18 @@ class ORunner(object):
def runWithInputUnit(self, inputUnit):
idx = 0
result = self.reporterFactory.create()
-
+ log.debug("Running test with input unit %s" % inputUnit)
for inputs in inputUnit:
result.reporterFactory = self.reporterFactory
+ log.debug("Running with %s" % inputs)
suite = self.baseSuite(self.cases)
suite.input = inputs
- suite(result, idx)
+ try:
+ suite(result, idx)
+ except Exception, e:
+ log.err("Error in running test!")
+ log.err(e)
# XXX refactor all of this index bullshit to avoid having to pass
# this index around. Probably what I want to do is go and make
@@ -250,7 +255,9 @@ class ORunner(object):
# We currently need to do this addition in order to get the number
# of times the test cases that have run inside of the test suite.
idx += (suite._idx - idx)
+ log.debug("I am now at the index %s" % idx)
+ log.debug("Finished")
result.done()
def run(self):
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 6e3163b..f453c74 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -54,6 +54,7 @@ class HTTPTest(TestCase):
followRedirects = False
def setUp(self):
+ log.debug("Setting up HTTPTest")
try:
import OpenSSL
except:
@@ -76,12 +77,17 @@ class HTTPTest(TestCase):
self.request = {}
self.response = {}
+ log.debug("Finished test setup")
- def _processResponseBody(self, data):
+ def _processResponseBody(self, data, body_processor):
+ log.debug("Processing response body")
self.response['body'] = data
self.report['response'] = self.response
- self.processResponseBody(data)
+ if body_processor:
+ body_processor(data)
+ else:
+ self.processResponseBody(data)
def processResponseBody(self, data):
"""
@@ -108,7 +114,25 @@ class HTTPTest(TestCase):
"""
pass
- def doRequest(self, url, method="GET", headers=None, body=None):
+ def doRequest(self, url, method="GET",
+ headers=None, body=None, headers_processor=None,
+ body_processor=None):
+ """
+ Perform an HTTP request with the specified method.
+
+ url: the full url path of the request
+ method: the HTTP Method to be used
+ headers: the request headers to be sent
+ body: the request body
+ headers_processor: a function to be used for processing the HTTP header
+ responses (defaults to self.processResponseHeaders).
+ This function takes as argument the HTTP headers as a
+ dict.
+ body_processory: a function to be used for processing the HTTP response
+ body (defaults to self.processResponseBody).
+ This function takes the response body as an argument.
+ """
+ log.debug("Performing request %s %s %s" % (url, method, headers))
try:
d = self.build_request(url, method, headers, body)
except Exception, e:
@@ -123,11 +147,17 @@ class HTTPTest(TestCase):
return
d.addErrback(errback)
- d.addCallback(self._cbResponse)
+ d.addCallback(self._cbResponse, headers_processor, body_processor)
d.addCallback(finished)
return d
- def _cbResponse(self, response):
+ def _cbResponse(self, response, headers_processor, body_processor):
+ log.debug("Got response %s" % response)
+ if not response:
+ self.report['response'] = None
+ log.err("We got an empty response")
+ return
+
self.response['headers'] = list(response.headers.getAllRawHeaders())
self.response['code'] = response.code
self.response['length'] = response.length
@@ -136,11 +166,14 @@ class HTTPTest(TestCase):
if str(self.response['code']).startswith('3'):
self.processRedirect(response.headers.getRawHeaders('Location')[0])
- self.processResponseHeaders(self.response['headers'])
+ if headers_processor:
+ headers_processor(self.response['headers'])
+ else:
+ self.processResponseHeaders(self.response['headers'])
finished = defer.Deferred()
response.deliverBody(BodyReceiver(finished))
- finished.addCallback(self._processResponseBody)
+ finished.addCallback(self._processResponseBody, body_processor)
return finished
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 38239ba..cd82ab4 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -4,6 +4,9 @@
import imp
import logging
+import string
+import random
+
try:
import yaml
except:
@@ -143,3 +146,32 @@ class Log():
except:
raise StopIteration
+def randomSTR(length, num=True):
+ """
+ Returns a random all uppercase alfa-numerical (if num True) string long length
+ """
+ chars = string.ascii_uppercase
+ if num:
+ chars += string.digits
+ return ''.join(random.choice(chars) for x in range(length))
+
+def randomstr(length, num=True):
+ """
+ Returns a random all lowercase alfa-numerical (if num True) string long length
+ """
+ chars = string.ascii_lowercase
+ if num:
+ chars += string.digits
+ return ''.join(random.choice(chars) for x in range(length))
+
+def randomStr(length, num=True):
+ """
+ Returns a random a mixed lowercase, uppercase, alfanumerical (if num True)
+ string long length
+ """
+ chars = string.ascii_lowercase + string.ascii_uppercase
+ if num:
+ chars += string.digits
+ return ''.join(random.choice(chars) for x in range(length))
+
+
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits