[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Fix date structure in reporting sytem.



commit 43cf06c1d2c5aa410aa8b220463d84f6ac7f0a26
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Tue Oct 2 23:45:07 2012 +0000

    Fix date structure in reporting sytem.
    * Collect errors in httpt template
    * Rename testhelpers to avoid namespace conflicts
    * Miscelaneous bug fixing
---
 bin/ooniprobe              |    2 +-
 nettests/example_httpt.py  |    9 ++++-
 nettests/example_scapyt.py |    2 +-
 ooni/lib/txscapy.py        |    2 +-
 ooni/nettest.py            |    7 ++-
 ooni/reporter.py           |   20 +++++----
 ooni/runner.py             |    6 ++-
 ooni/templates/httpt.py    |   28 ++++++-------
 ooni/utils/date.py         |    4 +-
 ooni/utils/log.py          |    2 +-
 oonib/oonibackend.py       |    5 ++-
 oonib/testhelpers/http.py  |   74 -----------------------------------
 oonib/testhelpers/httph.py |   92 ++++++++++++++++++++++++++++++++++++++++++++
 13 files changed, 143 insertions(+), 110 deletions(-)

diff --git a/bin/ooniprobe b/bin/ooniprobe
index c653756..4e400ce 100755
--- a/bin/ooniprobe
+++ b/bin/ooniprobe
@@ -1,4 +1,4 @@
-#!/usr/bin/python2
+#!/usr/bin/env python
 # -*- encoding: utf-8 -*-
 #
 # startup script based on twisted trial
diff --git a/nettests/example_httpt.py b/nettests/example_httpt.py
index 6501ca7..b113b23 100644
--- a/nettests/example_httpt.py
+++ b/nettests/example_httpt.py
@@ -10,7 +10,14 @@ class ExampleHTTP(httpt.HTTPTest):
     version = 0.1
 
     inputs = ['http://google.com/', 'http://wikileaks.org/',
-            'http://torproject.org/']
+              'http://torproject.org/']
+
+    def test_http(self):
+        if self.input:
+            url = self.input
+            return self.doRequest(url)
+        else:
+            raise Exception("No input specified")
 
     def processResponseBody(self, body):
         # XXX here shall go your logic
diff --git a/nettests/example_scapyt.py b/nettests/example_scapyt.py
index c7f342d..1cc8054 100644
--- a/nettests/example_scapyt.py
+++ b/nettests/example_scapyt.py
@@ -11,5 +11,5 @@ class ExampleScapy(scapyt.ScapyTest):
     version = 0.1
 
     inputs = [IP(dst="8.8.8.8")/TCP(dport=31337),
-            IP(dst="ooni.nu")/TCP(dport=31337)]
+              IP(dst="ooni.nu")/TCP(dport=31337)]
 
diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py
index ec605e5..d4f146e 100644
--- a/ooni/lib/txscapy.py
+++ b/ooni/lib/txscapy.py
@@ -84,7 +84,7 @@ class Scapy(object):
     """
     min = 2
     max = 6
-    debug = True
+    debug = False
     write_only_answers = False
     pcapwriter = None
     recv = False
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 403a022..d7188c6 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -81,8 +81,11 @@ class TestCase(unittest.TestCase):
         if self.inputFile:
             fp = open(self.inputFile)
             self.inputs = inputProcessor(fp)
-
-        return {'inputs': self.inputs}
+        # XXX perhaps we may want to name and version to be inside of a
+        # different object that is not called options.
+        return {'inputs': self.inputs,
+                'name': self.name,
+                'version': self.version}
 
     def __repr__(self):
         return "<%s inputs=%s>" % (self.__class__, self.inputs)
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 51a0166..3e108f9 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -7,6 +7,7 @@ sys.path.insert(0, '/home/x/Documents/pen_drive_bitcoin2012/ooni-probe/ENV/lib/p
 from datetime import datetime
 from twisted.python.util import OrderedDict, untilConcludes
 from twisted.trial import unittest, reporter, runner
+from ooni.utils import date
 
 try:
     from scapy.all import packet
@@ -70,19 +71,19 @@ class ReporterFactory(OReporter):
         self._testSuite = testSuite
         self._reporters = []
 
-    def writeHeader(self):
-        pretty_date = "XXX Replace me with date.pretty_date()"
+    def writeHeader(self, options):
         self._writeln("###########################################")
-        self._writeln("# OONI Probe Report for Test %s" % "XXX replace with with the test suite name")
-        self._writeln("# %s" % pretty_date)
+        self._writeln("# OONI Probe Report for %s test" % options['name'])
+        self._writeln("# %s" % date.pretty_date())
         self._writeln("###########################################")
 
         address = {'asn': 'XXX replace me with ASN',
                    'ip': 'XXX replace me with IP'}
-        test_details = {'start_time': datetime.now(),
-                        'asn': address['asn'],
-                        'test_name': 'XXX replace me with the test name',
-                        'addr': address['ip']}
+        test_details = {'startTime': repr(date.now()),
+                        'probeASN': address['asn'],
+                        'testName': options['name'],
+                        'testVersion': options['version'],
+                        'probeIP': address['ip']}
         self.writeYamlLine(test_details)
         self._writeln('')
 
@@ -135,8 +136,9 @@ class OONIReporter(OReporter):
             test_input = repr(test.input)
         else:
             test_input = test.input
+
         self._tests[idx]['input'] = test_input
-        self._tests[idx]['idx'] = idx
+        #self._tests[idx]['idx'] = idx
         self._tests[idx]['name'] = test.name
         #self._tests[idx]['test'] = test
         print "Now starting %s" % self._tests[idx]
diff --git a/ooni/runner.py b/ooni/runner.py
index 7883f07..0738bf8 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -129,6 +129,8 @@ def loadTestsAndOptions(classes):
     suiteFactory = InputTestSuite
     options = []
     testCases = []
+    names = []
+
     for klass in classes:
         try:
             k = klass()
@@ -184,8 +186,8 @@ class ORunner(object):
 
     def run(self):
         #log.startLogging(sys.stdout)
-        log.start(True)
-        self.reporterFactory.writeHeader()
+        log.start()
+        self.reporterFactory.writeHeader(self.options)
 
         for inputUnit in InputUnitFactory(self.inputs):
             self.runWithInputUnit(inputUnit)
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 51775f0..a537c6e 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -106,27 +106,25 @@ class HTTPTest(TestCase):
         """
         pass
 
-    def doRequest(self, url):
-        d = self.build_request(url)
+    def doRequest(self, url, method="GET", headers=None, body=None):
+        try:
+            d = self.build_request(url, method, headers, body)
+        except Exception, e:
+            print e
+            self.report['error'] = e
+
+        def errback(data):
+            print data
+            #self.report["error"] = data
+
         def finished(data):
-            #self.mainDefer.callback()
-            return data
+            return
 
+        d.addErrback(errback)
         d.addCallback(self._cbResponse)
         d.addCallback(finished)
         return d
 
-    def test_http(self):
-        log.msg("Running experiment")
-
-        if self.input:
-            url = self.input
-        else:
-            raise Exception("No input supplied")
-
-        self.mainDefer = self.doRequest(url)
-        return self.mainDefer
-
     def _cbResponse(self, response):
         self.response['headers'] = response.headers
         self.response['code'] = response.code
diff --git a/ooni/utils/date.py b/ooni/utils/date.py
index 59ec1f8..dcd8e36 100644
--- a/ooni/utils/date.py
+++ b/ooni/utils/date.py
@@ -3,10 +3,10 @@ from datetime import datetime
 
 class odate(datetime):
     def __str__(self):
-        return rfc3339(self)
+        return "%s" % rfc3339(self)
 
     def __repr__(self):
-        return "'%s'" % rfc3339(self)
+        return "%s" % rfc3339(self)
 
     def from_rfc(self, datestr):
         pass
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 6ff6bf9..06bc3fa 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -73,7 +73,7 @@ class OONILogObserver(log.FileLogObserver):
         util.untilConcludes(self.write, timeStr + " " + msgStr)
         util.untilConcludes(self.flush)
 
-def start(logstdout, logfile=None, verbosity=None):
+def start(logfile=None, logstdout=True, verbosity=None):
     if log.defaultObserver:
         verbosity = _get_log_level(verbosity)
 
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index d6ba3f0..0232829 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -17,8 +17,8 @@ from twisted.web.microdom import escape
 from twisted.names import dns
 
 from oonib.lib import config
-from oonib.testhelpers.http import HTTPBackend
 from oonib.lib.ssl import SSLContext
+from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer
 from oonib.testhelpers.dns import ProxyDNSServer
 from oonib.testhelpers.daphn3 import Daphn3Server
 
@@ -37,6 +37,9 @@ internet.SSLServer(int(config.main.ssl_port),
                    SSLContext(config),
                   ).setServiceParent(serviceCollection)
 
+debugHTTPServer = DebugHTTPServer()
+internet.TCPServer(8090, debugHTTPServer).setServiceParent(serviceCollection)
+
 # Start the DNS Server related services
 TCPDNSServer = ProxyDNSServer()
 internet.TCPServer(int(config.main.dns_tcp_port), TCPDNSServer).setServiceParent(serviceCollection)
diff --git a/oonib/testhelpers/http.py b/oonib/testhelpers/http.py
deleted file mode 100644
index 79a487b..0000000
--- a/oonib/testhelpers/http.py
+++ /dev/null
@@ -1,74 +0,0 @@
-import json
-import random
-import string
-
-from twisted.application import internet, service
-from twisted.internet import protocol, reactor, defer
-from twisted.protocols import basic
-from twisted.web import resource, server, static
-from twisted.web.microdom import escape
-
-server.version = "Apache"
-
-class HTTPRandomPage(resource.Resource):
-    """
-    This generates a random page of arbitrary length and containing the string
-    selected by the user.
-    The format is the following:
-    /random/<length>/<keyword>
-    """
-    isLeaf = True
-    def _gen_random_string(self, length):
-        return ''.join(random.choice(string.letters) for x in range(length))
-
-    def genRandomPage(self, length=100, keyword=None):
-        data = self._gen_random_string(length/2)
-        if keyword:
-            data += keyword
-        data += self._gen_random_string(length - length/2)
-        data += '\n'
-        return data
-
-    def render(self, request):
-        length = 100
-        keyword = None
-        path_parts = request.path.split('/')
-        if len(path_parts) > 2:
-            length = int(path_parts[2])
-            if length > 100000:
-                length = 100000
-
-        if len(path_parts) > 3:
-            keyword = escape(path_parts[3])
-
-        return self.genRandomPage(length, keyword)
-
-class HTTPReturnHeaders(resource.Resource):
-    """
-    This returns the headers being sent by the client in JSON format.
-    """
-    isLeaf = True
-    def render(self, request):
-        req_headers = request.getAllHeaders()
-        return json.dumps(req_headers)
-
-class HTTPSendHeaders(resource.Resource):
-    """
-    This sends to the client the headers that they send inside of the POST
-    request encoded in json.
-    """
-    isLeaf = True
-    def render_POST(self, request):
-        headers = json.loads(request.content.read())
-        for header, value in headers.items():
-            request.setHeader(str(header), str(value))
-        return ''
-
-class HTTPBackend(resource.Resource):
-    def __init__(self):
-        resource.Resource.__init__(self)
-        self.putChild('random', HTTPRandomPage())
-        self.putChild('returnheaders', HTTPReturnHeaders())
-        self.putChild('sendheaders', HTTPSendHeaders())
-
-
diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py
new file mode 100644
index 0000000..217dbd5
--- /dev/null
+++ b/oonib/testhelpers/httph.py
@@ -0,0 +1,92 @@
+import json
+import random
+import string
+
+from twisted.application import internet, service
+from twisted.internet import protocol, reactor, defer
+from twisted.protocols import basic
+from twisted.web import resource, server, static, http
+from twisted.web.microdom import escape
+
+server.version = "Apache"
+
+class HTTPRandomPage(resource.Resource):
+    """
+    This generates a random page of arbitrary length and containing the string
+    selected by the user.
+    The format is the following:
+    /random/<length>/<keyword>
+    """
+    isLeaf = True
+    def _gen_random_string(self, length):
+        return ''.join(random.choice(string.letters) for x in range(length))
+
+    def genRandomPage(self, length=100, keyword=None):
+        data = self._gen_random_string(length/2)
+        if keyword:
+            data += keyword
+        data += self._gen_random_string(length - length/2)
+        data += '\n'
+        return data
+
+    def render(self, request):
+        length = 100
+        keyword = None
+        path_parts = request.path.split('/')
+        if len(path_parts) > 2:
+            length = int(path_parts[2])
+            if length > 100000:
+                length = 100000
+
+        if len(path_parts) > 3:
+            keyword = escape(path_parts[3])
+
+        return self.genRandomPage(length, keyword)
+
+class HTTPReturnHeaders(resource.Resource):
+    """
+    This returns the headers being sent by the client in JSON format.
+    """
+    isLeaf = True
+    def render(self, request):
+        req_headers = request.getAllHeaders()
+        return json.dumps(req_headers)
+
+class HTTPSendHeaders(resource.Resource):
+    """
+    This sends to the client the headers that they send inside of the POST
+    request encoded in json.
+    """
+    isLeaf = True
+    def render_POST(self, request):
+        headers = json.loads(request.content.read())
+        for header, value in headers.items():
+            request.setHeader(str(header), str(value))
+        return ''
+
+class HTTPBackend(resource.Resource):
+    def __init__(self):
+        resource.Resource.__init__(self)
+        self.putChild('random', HTTPRandomPage())
+        self.putChild('returnheaders', HTTPReturnHeaders())
+        self.putChild('sendheaders', HTTPSendHeaders())
+
+class DebugProtocol(http.HTTPChannel):
+    def headerReceived(self, line):
+        print "[HEADER] %s" % line
+        http.HTTPChannel.headerReceived(self, line)
+
+    def allContentReceived(self):
+        print self.requests[-1].getAllHeaders()
+        self.transport.loseConnection()
+        self.connectionLost("Normal closure")
+
+class DebugHTTPServer(http.HTTPFactory):
+    protocol = DebugProtocol
+
+    def buildProtocol(self, addr):
+        print "Got connection from %s" % addr
+        p = protocol.ServerFactory.buildProtocol(self, addr)
+        p.timeOut = self.timeOut
+        return p
+



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits