[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Refactor reporter Object.



commit 572b301710f395620c557dde8ecdef0771ccbc88
Author: Arturo Filastò <art@xxxxxxxxx>
Date:   Sat Nov 10 23:32:48 2012 +0100

    Refactor reporter Object.
    * We now have a parent OReporter object that is subclassed by OONIBReporter for
      remote reporting and YAMLReporter for reporting to YAML format on file system.
    * Move secure YAML serialization hacks to the hacks module
    * Do more progress on the implementation of reporting to remote systems
---
 ooni/reporter.py    |  282 ++++++++++++++++++++++++++-------------------------
 ooni/runner.py      |    8 +-
 ooni/utils/hacks.py |   47 +++++++++
 3 files changed, 197 insertions(+), 140 deletions(-)

diff --git a/ooni/reporter.py b/ooni/reporter.py
index 5d29d54..b625603 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -26,6 +26,8 @@ from twisted.internet import defer, reactor
 
 from ooni.templates.httpt import BodyReceiver, StringProducer
 from ooni.utils import otime, log, geodata
+
+from ooni.utils.hacks import OSafeRepresenter, OSafeDumper
 from ooni import config
 
 try:
@@ -38,47 +40,8 @@ except:
         Packet = object
     packet = FooClass
 
-class OSafeRepresenter(SafeRepresenter):
-    """
-    This is a custom YAML representer that allows us to represent reports
-    safely.
-    It extends the SafeRepresenter to be able to also represent complex numbers
-    """
-    def represent_complex(self, data):
-        if data.imag == 0.0:
-            data = u'%r' % data.real
-        elif data.real == 0.0:
-            data = u'%rj' % data.imag
-        elif data.imag > 0:
-            data = u'%r+%rj' % (data.real, data.imag)
-        else:
-            data = u'%r%rj' % (data.real, data.imag)
-        return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data)
-
-OSafeRepresenter.add_representer(complex,
-                                 OSafeRepresenter.represent_complex)
-
-class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver):
-    """
-    This is a modification of the YAML Safe Dumper to use our own Safe
-    Representer that supports complex numbers.
-    """
-    def __init__(self, stream,
-            default_style=None, default_flow_style=None,
-            canonical=None, indent=None, width=None,
-            allow_unicode=None, line_break=None,
-            encoding=None, explicit_start=None, explicit_end=None,
-            version=None, tags=None):
-        Emitter.__init__(self, stream, canonical=canonical,
-                indent=indent, width=width,
-                allow_unicode=allow_unicode, line_break=line_break)
-        Serializer.__init__(self, encoding=encoding,
-                explicit_start=explicit_start, explicit_end=explicit_end,
-                version=version, tags=tags)
-        OSafeRepresenter.__init__(self, default_style=default_style,
-                default_flow_style=default_flow_style)
-        Resolver.__init__(self)
-
+class NoTestIDSpecified(Exception):
+    pass
 
 def safe_dump(data, stream=None, **kw):
     """
@@ -86,42 +49,93 @@ def safe_dump(data, stream=None, **kw):
     """
     return yaml.dump_all([data], stream, Dumper=OSafeDumper, **kw)
 
-class OONIBReporter(object):
-    def __init__(self, backend_url):
-        from twisted.web.client import Agent
-        from twisted.internet import reactor
+@xxxxxxxxxxxxxxxxxxxxx
+def getTestDetails(options):
+    client_geodata = {}
+
+    if config.privacy.includeip or \
+            config.privacy.includeasn or \
+            config.privacy.includecountry or \
+            config.privacy.includecity:
+        log.msg("Running geo IP lookup via check.torproject.org")
+        client_ip = yield geodata.myIP()
+        client_location = geodata.IPToLocation(client_ip)
+    else:
+        client_ip = "127.0.0.1"
+
+    if config.privacy.includeip:
+        client_geodata['ip'] = client_ip
+    else:
+        client_geodata['ip'] = "127.0.0.1"
+
+    client_geodata['asn'] = None
+    client_geodata['city'] = None
+    client_geodata['countrycode'] = None
+
+    if config.privacy.includeasn:
+        client_geodata['asn'] = client_location['asn']
+
+    if config.privacy.includecity:
+        client_geodata['city'] = client_location['city']
+
+    if config.privacy.includecountry:
+        client_geodata['countrycode'] = client_location['countrycode']
+
+    test_details = {'start_time': otime.utcTimeNow(),
+                    'probe_asn': client_geodata['asn'],
+                    'probe_cc': client_geodata['countrycode'],
+                    'probe_ip': client_geodata['ip'],
+                    'test_name': options['name'],
+                    'test_version': options['version'],
+                    }
+    defer.returnValue(test_details)
+    
+
+class OReporter(object):
+    def createReport(options):
+        """
+        Override this with your own logic to implement tests.
+        """
+        raise NotImplemented
 
-        self.agent = Agent(reactor)
-        self.backend_url = backend_url
+    def writeReportEntry(self, entry):
+        """
+        Takes as input an entry and writes a report for it.
+        """
+        raise NotImplemented
 
-    def _newReportCreated(self, data):
-        log.debug("Got this as result: %s" % data)
-        return data
+    def finish():
+        pass
 
-    def _processResponseBody(self, response, body_cb):
-        log.debug("Got response %s" % response)
-        done = defer.Deferred()
-        response.deliverBody(BodyReceiver(done))
-        done.addCallback(body_cb)
-        return done
+    def testDone(self, test, test_name):
+        test_report = dict(test.report)
 
-    def newReport(self, test_name, test_version):
-        url = self.backend_url + '/new'
-        software_version = '0.0.1'
+        # XXX the scapy test has an example of how 
+        # to do this properly.
+        if isinstance(test.input, packet.Packet):
+            test_input = repr(test.input)
+        else:
+            test_input = test.input
 
-        request = {'software_name': 'ooni-probe',
-                'software_version': software_version,
-                'test_name': test_name, 'test_version': test_version,
-                'progress': 0}
+        test_started = test._start_time
+        test_runtime = test_started - time.time()
 
-        log.debug("Creating report via url %s" % url)
-        bodyProducer = StringProducer(json.dumps(request))
-        d = self.agent.request("POST", url, bodyProducer=bodyProducer)
-        d.addCallback(self._processResponseBody, self._newReportCreated)
-        return d
+        report = {'input': test_input,
+                'test_name': test_name,
+                'test_started': test_started,
+                'report': test_report}
+        self.writeReportEntry(report)
 
+    def allDone(self):
+        log.debug("allDone: Finished running all tests")
+        self.finish()
+        try:
+            reactor.stop()
+        except:
+            pass
+        return None
 
-class YamlReporter(object):
+class YAMLReporter(OReporter):
     """
     These are useful functions for reporting to YAML format.
     """
@@ -145,91 +159,85 @@ class YamlReporter(object):
         self._write(safe_dump(entry))
         self._write('...\n')
 
-    def finish(self):
-        self._stream.close()
-
-class OReporter(YamlReporter):
-    """
-    This is a reporter factory. It emits new instances of Reports. It is also
-    responsible for writing the OONI Report headers.
-    """
-    def writeTestsReport(self, tests):
-        for test in tests.values():
-            self.writeReportEntry(test)
-
     @defer.inlineCallbacks
-    def writeReportHeader(self, options):
-        self.firstrun = False
+    def createReport(self, options):
         self._writeln("###########################################")
         self._writeln("# OONI Probe Report for %s test" % options['name'])
         self._writeln("# %s" % otime.prettyDateNow())
         self._writeln("###########################################")
 
-        client_geodata = {}
+        test_details = yield getTestDetails(options)
 
-        if config.privacy.includeip or \
-                config.privacy.includeasn or \
-                config.privacy.includecountry or \
-                config.privacy.includecity:
-            log.msg("Running geo IP lookup via check.torproject.org")
-            client_ip = yield geodata.myIP()
-            client_location = geodata.IPToLocation(client_ip)
-        else:
-            client_ip = "127.0.0.1"
+        self.writeReportEntry(test_details)
 
-        if config.privacy.includeip:
-            client_geodata['ip'] = client_ip
-        else:
-            client_geodata['ip'] = "127.0.0.1"
+    def finish(self):
+        self._stream.close()
 
-        client_geodata['asn'] = None
-        client_geodata['city'] = None
-        client_geodata['countrycode'] = None
+class OONIBReporter(object):
+    def __init__(self, backend_url):
+        from twisted.web.client import Agent
+        from twisted.internet import reactor
+        self.agent = Agent(reactor)
+        self.backend_url = backend_url
 
-        if config.privacy.includeasn:
-            client_geodata['asn'] = client_location['asn']
+    def _newReportCreated(self, data):
+        log.debug("newReportCreated %s" % data)
+        return data
 
-        if config.privacy.includecity:
-            client_geodata['city'] = client_location['city']
+    def _processResponseBody(self, response, body_cb):
+        log.debug("processResponseBody %s" % response)
+        done = defer.Deferred()
+        response.deliverBody(BodyReceiver(done))
+        done.addCallback(body_cb)
+        return done
 
-        if config.privacy.includecountry:
-            client_geodata['countrycode'] = client_location['countrycode']
+    def createReport(self, test_name,
+            test_version, report_header):
 
+        url = self.backend_url + '/new'
+        software_version = '0.0.1'
 
-        test_details = {'start_time': otime.utcTimeNow(),
-                        'probe_asn': client_geodata['asn'],
-                        'probe_cc': client_geodata['countrycode'],
-                        'probe_ip': client_geodata['ip'],
-                        'test_name': options['name'],
-                        'test_version': options['version'],
-                        }
-        self.writeReportEntry(test_details)
+        request = {'software_name': 'ooni-probe',
+                'software_version': software_version,
+                'test_name': test_name,
+                'test_version': test_version,
+                'progress': 0,
+                'content': report_header
+        }
+        def gotDetails(test_details):
+            log.debug("Creating report via url %s" % url)
+
+            bodyProducer = StringProducer(json.dumps(request))
+            d = self.agent.request("POST", url, 
+                    bodyProducer=bodyProducer)
+            d.addCallback(self._processResponseBody, 
+                    self._newReportCreated)
+            return d
+
+        d = getTestDetails(options)
+        d.addCallback(gotDetails)
+        return d
 
-    def testDone(self, test, test_name):
-        test_report = dict(test.report)
+    def writeReportEntry(self, entry, test_id=None):
+        if not test_id:
+            log.err("Write report entry on OONIB requires test id")
+            raise NoTestIDSpecified
 
-        # XXX the scapy test has an example of how 
-        # to do this properly.
-        if isinstance(test.input, packet.Packet):
-            test_input = repr(test.input)
-        else:
-            test_input = test.input
+        report = '---\n'
+        report += safe_dump(entry)
+        report += '...\n'
 
-        test_started = test._start_time
-        test_runtime = test_started - time.time()
+        url = self.backend_url + '/new'
 
-        report = {'input': test_input,
-                'test_name': test_name,
-                'test_started': test_started,
-                'report': test_report}
-        self.writeReportEntry(report)
+        request = {'test_id': test_id,
+                'content': report}
+
+        bodyProducer = StringProducer(json.dumps(request))
+        d = self.agent.request("PUT", url,
+                bodyProducer=bodyProducer)
+
+        d.addCallback(self._processResponseBody,
+                    self._newReportCreated)
+        return d
 
-    def allDone(self):
-        log.debug("allDone: Finished running all tests")
-        self.finish()
-        try:
-            reactor.stop()
-        except:
-            pass
-        return None
 
diff --git a/ooni/runner.py b/ooni/runner.py
index 1affb15..07882cf 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -194,7 +194,8 @@ def runTestWithInputUnit(test_class,
     for test_input in input_unit:
         log.debug("IU: %s" % test_input)
         try:
-            d = runTestWithInput(test_class, test_method, test_input, oreporter)
+            d = runTestWithInput(test_class, 
+                    test_method, test_input, oreporter)
         except Exception, e:
             print e
         log.debug("here y0")
@@ -223,10 +224,11 @@ def runTestCases(test_cases, options,
             test_inputs = [None]
 
     reportFile = open(yamloo_filename, 'w+')
-    oreporter = reporter.OReporter(reportFile)
+    oreporter = reporter.YAMLReporter(reportFile)
+
     input_unit_factory = InputUnitFactory(test_inputs)
 
-    yield oreporter.writeReportHeader(options)
+    yield oreporter.createReport(options)
     # This deferred list is a deferred list of deferred lists
     # it is used to store all the deferreds of the tests that 
     # are run
diff --git a/ooni/utils/hacks.py b/ooni/utils/hacks.py
index 4eef366..8912bbb 100644
--- a/ooni/utils/hacks.py
+++ b/ooni/utils/hacks.py
@@ -8,6 +8,11 @@
 # :authors: Arturo Filastò
 # :licence: see LICENSE
 
+from yaml.representer import *
+from yaml.emitter import *
+from yaml.serializer import *
+from yaml.resolver import *
+
 import copy_reg
 
 def patched_reduce_ex(self, proto):
@@ -61,3 +66,45 @@ def patched_reduce_ex(self, proto):
         return copy_reg._reconstructor, args, dict
     else:
         return copy_reg._reconstructor, args
+
+class OSafeRepresenter(SafeRepresenter):
+    """
+    This is a custom YAML representer that allows us to represent reports
+    safely.
+    It extends the SafeRepresenter to be able to also represent complex numbers
+    """
+    def represent_complex(self, data):
+        if data.imag == 0.0:
+            data = u'%r' % data.real
+        elif data.real == 0.0:
+            data = u'%rj' % data.imag
+        elif data.imag > 0:
+            data = u'%r+%rj' % (data.real, data.imag)
+        else:
+            data = u'%r%rj' % (data.real, data.imag)
+        return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data)
+
+OSafeRepresenter.add_representer(complex,
+                                 OSafeRepresenter.represent_complex)
+
+class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver):
+    """
+    This is a modification of the YAML Safe Dumper to use our own Safe
+    Representer that supports complex numbers.
+    """
+    def __init__(self, stream,
+            default_style=None, default_flow_style=None,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None):
+        Emitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width,
+                allow_unicode=allow_unicode, line_break=line_break)
+        Serializer.__init__(self, encoding=encoding,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        OSafeRepresenter.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style)
+        Resolver.__init__(self)
+



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits