[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Merge branch 'master' into remotereport



commit fb7c4962063161e621691b128f23d70402c4da36
Merge: 10f0d90 9e7cf8c
Author: Arturo Filastò <art@xxxxxxxxx>
Date:   Sun Nov 11 16:48:30 2012 +0100

    Merge branch 'master' into remotereport
    
    * master:
      Implement basic keyword filtering detection test. * XXX Does not use the bisection method * XXX Does not establish a TCP session, but just sends
      Do some major refactoring of the ooniprobe backend * Implement TCP Echo test helper * Refactor testhelpers into proper namespace * Remove unused imports * Remove files that were not being imported anywhere
      Respect the includeip = false option in ooniprobe.conf for scapyt derivate tests * If the option to not include the IP address of the probe is set, change the   source and destination ip address of the sent and received packets to   127.0.0.1
      Add ooniprobe prefix to logs
      Clean up filenames of generated reports * They now follow the format $testName_report_$timestamp.yamloo
      Implement ICMP traceroute test * parametrize max_ttl and timeout * make max_ttl and timeout be included in the report
      Add not about the fact that tests should return a deferred.
      Add software version to the report * Cleanup some debug messages * Remove unused ooni-probe.conf config file
      Implement basic reporting to flat files containing the report ID. * XXX the location of such files is not configurable, they will leave tons of   crap in your cwd.
      Refactor reporter Object. * We now have a parent OReporter object that is subclassed by OONIBReporter for   remote reporting and YAMLReporter for reporting to YAML format on file system. * Move secure YAML serialization hacks to the hacks module * Do more progress on the implementation of reporting to remote systems
    
    Conflicts:
    	ooni/reporter.py
    	ooni/runner.py

 nettests/core/keyword_filtering.py |   52 +++++++
 nettests/core/squid.py             |    2 +-
 nettests/core/traceroute.py        |   70 ++++++++--
 ooni/__init__.py                   |    4 +
 ooni/config.py                     |    9 +-
 ooni/inputunit.py                  |    2 +-
 ooni/nettest.py                    |    3 +
 ooni/ooni-probe.conf               |  100 --------------
 ooni/oonicli.py                    |    8 +-
 ooni/reporter.py                   |  265 ++++++++++++++++++------------------
 ooni/runner.py                     |   14 ++-
 ooni/templates/scapyt.py           |   18 ++-
 ooni/utils/hacks.py                |   47 +++++++
 ooni/utils/log.py                  |    4 +-
 oonib/config.py                    |   39 ++++--
 oonib/lib/__init__.py              |    1 -
 oonib/lib/ssl.py                   |    8 -
 oonib/oonibackend.conf.sample      |   12 --
 oonib/oonibackend.py               |   79 ++++++-----
 oonib/report/api.py                |  123 +++++++++++++++--
 oonib/testhelpers/__init__.py      |    5 +
 oonib/testhelpers/dns.py           |   16 --
 oonib/testhelpers/dns_helpers.py   |   16 ++
 oonib/testhelpers/http_helpers.py  |   80 +++++++++++
 oonib/testhelpers/httph.py         |   80 -----------
 oonib/testhelpers/ssl_helpers.py   |    9 ++
 oonib/testhelpers/tcp_helpers.py   |   14 ++
 ooniprobe.conf                     |    3 +-
 28 files changed, 646 insertions(+), 437 deletions(-)

diff --cc nettests/core/squid.py
index 675119c,675119c..777bc3e
--- a/nettests/core/squid.py
+++ b/nettests/core/squid.py
@@@ -17,7 -17,7 +17,7 @@@ class SquidTest(httpt.HTTPTest)
      """
      name = "Squid test"
      author = "Arturo Filastò"
--    version = 0.1
++    version = "0.1"
  
      optParameters = [['backend', 'b', 'http://ooni.nu/test/', 'Test backend to use']]
  
diff --cc ooni/reporter.py
index 31c53a9,b5e2039..7147b35
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@@ -168,122 -162,78 +162,129 @@@ class YAMLReporter(OReporter)
          self._writeln("# %s" % otime.prettyDateNow())
          self._writeln("###########################################")
  
-         client_geodata = {}
+         test_details = yield getTestDetails(options)
  
-         if config.privacy.includeip or \
-                 config.privacy.includeasn or \
-                 config.privacy.includecountry or \
-                 config.privacy.includecity:
-             log.msg("Running geo IP lookup via check.torproject.org")
-             client_ip = yield geodata.myIP()
-             client_location = geodata.IPToLocation(client_ip)
-         else:
-             client_ip = "127.0.0.1"
+         self.writeReportEntry(test_details)
  
-         if config.privacy.includeip:
-             client_geodata['ip'] = client_ip
-         else:
-             client_geodata['ip'] = "127.0.0.1"
+     def finish(self):
+         self._stream.close()
  
-         client_geodata['asn'] = None
-         client_geodata['city'] = None
-         client_geodata['countrycode'] = None
+ class OONIBReporter(object):
+     def __init__(self, backend_url):
+         from twisted.web.client import Agent
+         from twisted.internet import reactor
+         self.agent = Agent(reactor)
+         self.backend_url = backend_url
  
-         if config.privacy.includeasn:
-             client_geodata['asn'] = client_location['asn']
+     def _newReportCreated(self, data):
+         log.debug("newReportCreated %s" % data)
+         return data
  
-         if config.privacy.includecity:
-             client_geodata['city'] = client_location['city']
+     def _processResponseBody(self, response, body_cb):
+         log.debug("processResponseBody %s" % response)
+         done = defer.Deferred()
+         response.deliverBody(BodyReceiver(done))
+         done.addCallback(body_cb)
+         return done
  
-         if config.privacy.includecountry:
-             client_geodata['countrycode'] = client_location['countrycode']
+     def createReport(self, test_name,
+             test_version, report_header):
 -
+         url = self.backend_url + '/new'
+         software_version = '0.0.1'
  
-         test_details = {'start_time': otime.utcTimeNow(),
-                         'probe_asn': client_geodata['asn'],
-                         'probe_cc': client_geodata['countrycode'],
-                         'probe_ip': client_geodata['ip'],
-                         'test_name': options['name'],
-                         'test_version': options['version'],
-                         }
-         self.writeReportEntry(test_details)
+         request = {'software_name': 'ooni-probe',
+                 'software_version': software_version,
+                 'test_name': test_name,
+                 'test_version': test_version,
+                 'progress': 0,
+                 'content': report_header
+         }
+         def gotDetails(test_details):
+             log.debug("Creating report via url %s" % url)
+ 
+             bodyProducer = StringProducer(json.dumps(request))
+             d = self.agent.request("POST", url, 
+                     bodyProducer=bodyProducer)
+             d.addCallback(self._processResponseBody, 
+                     self._newReportCreated)
+             return d
+ 
+         d = getTestDetails(options)
+         d.addCallback(gotDetails)
+         return d
+ 
+     def writeReportEntry(self, entry, test_id=None):
+         if not test_id:
+             log.err("Write report entry on OONIB requires test id")
+             raise NoTestIDSpecified
+ 
+         report = '---\n'
+         report += safe_dump(entry)
+         report += '...\n'
  
-     def testDone(self, test, test_name):
-         test_report = dict(test.report)
+         url = self.backend_url + '/new'
  
-         # XXX the scapy test has an example of how 
-         # to do this properly.
-         if isinstance(test.input, packet.Packet):
-             test_input = repr(test.input)
-         else:
-             test_input = test.input
+         request = {'test_id': test_id,
+                 'content': report}
  
-         test_started = test._start_time
-         test_runtime = test_started - time.time()
+         bodyProducer = StringProducer(json.dumps(request))
+         d = self.agent.request("PUT", url,
+                 bodyProducer=bodyProducer)
  
-         report = {'input': test_input,
-                 'test_name': test_name,
-                 'test_started': test_started,
-                 'report': test_report}
-         self.writeReportEntry(report)
+         d.addCallback(self._processResponseBody,
+                     self._newReportCreated)
+         return d
  
-     def allDone(self):
-         log.debug("allDone: Finished running all tests")
-         self.finish()
-         try:
-             reactor.stop()
-         except:
-             pass
-         return None
  
 +
 +class OONIBReporter(OReporter):
 +    def __init__(self, backend_url):
 +        from twisted.web.client import Agent
 +        from twisted.internet import reactor
 +        self.agent = Agent(reactor)
 +        self.backend_url = backend_url
 +
 +    def _processResponseBody(self, *arg, **kw):
 +        #done = defer.Deferred()
 +        #response.deliverBody(BodyReceiver(done))
 +        #done.addCallback(self._newReportCreated)
 +        #return done
 +
 +    def createReport(self, options):
 +        test_name = options['name']
 +        test_version = options['version']
 +
 +        log.debug("Creating report with OONIB Reporter")
 +        url = self.backend_url + '/report/new'
 +        software_version = '0.0.1'
 +
 +        def gotDetails(test_details):
 +            content = '---\n'
 +            content += safe_dump(test_details)
 +            content += '...\n'
 +
 +            request = {'software_name': 'ooniprobe',
 +                'software_version': software_version,
 +                'test_name': test_name,
 +                'test_version': test_version,
 +                'progress': 0,
 +                'content': content
 +            }
 +            log.debug("Creating report via url %s" % url)
 +            request_json = json.dumps(request)
 +            log.debug("Sending %s" % request_json)
 +
 +            def bothCalls(*arg, **kw):
 +                print arg, kw
 +
 +            body_producer = StringProducer(request_json)
 +            d = self.agent.request("POST", url, None,
 +                    body_producer)
 +            d.addBoth(self._processResponseBody)
 +            return d
 +
 +        d = getTestDetails(options)
 +        d.addCallback(gotDetails)
 +        # XXX handle errors
 +        return d
 +
diff --cc ooni/runner.py
index c2105ea,23ff898..b92ad7e
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@@ -221,16 -221,13 +221,22 @@@ def runTestCases(test_cases, options
              log.msg("Could not find inputs!")
              log.msg("options[0] = %s" % first)
              test_inputs = [None]
- 
+     
      reportFile = open(yamloo_filename, 'w+')
++
 +    #oreporter = reporter.YAMLReporter(reportFile)
 +    oreporter = reporter.OONIBReporter('http://127.0.0.1:8888')
 +
 +    input_unit_factory = InputUnitFactory(test_inputs)
 +
 +    log.debug("Creating report")
 +    yield oreporter.createReport(options)
 +
+     oreporter = reporter.YAMLReporter(reportFile)
+ 
+     input_unit_factory = InputUnitFactory(test_inputs)
+ 
+     yield oreporter.createReport(options)
      # This deferred list is a deferred list of deferred lists
      # it is used to store all the deferreds of the tests that 
      # are run



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits