[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Squashed commit of the following:



commit 6d2728ca795a97f6929c37579432ebcf909097d8
Author: Daniel Ramsay <daniel@xxxxxxxxxxxxx>
Date:   Mon Aug 31 23:14:52 2015 +0100

    Squashed commit of the following:
    
        Add support for ORG's blocked.org.uk scheduler
    
        Add:
        * ORG Queue integration
        * Support for additional values to send to ORG backend
    
        Refactor director startup
        * Moved collector setup to shared scope
        * Moved deck setup to shared scope
        * Moved annotations setup to shared scope
        * Moved global options to shared scope
    
        * fix some naming convention errors
    
        * move global counter to more sensible scope
        * fixes from pull request code review
        * remove another mobile browser
        * remove iPhone useragent to prevent mobile versions being returned to client
    
        * add HTTPS support for reporter
        * add authentication info from environment
        * import PROBE_ vars from environment into report
        * allow https collector in regex
        * AMQP connection details as cmdline parameter
        * more queue state, looping listen call
        * remove old bin script
        * exit after bounded random number of test decks
        * improve error handling
        * use proper custom exception for queue timeout reason
        * reduce duplicate serialization
        * add flag to prevent response body being written to report
        * add subargs to queue createdeck
        * fix global args passing
        * reuse cached geoip
        * reuse existing TOR connection
        * simplify for one-shot use
        * extend and report lifetime
        * probe exit on AMQP error
        * remove ooniresource from setup script
---
 Vagrantfile                             |   2 +
 bin/ooniprobequeue                      |  21 ++
 ooni/director.py                        |   2 +-
 ooni/geoip.py                           |  57 +++--
 ooni/nettests/blocking/http_requests.py |   3 +
 ooni/oonicli.py                         | 426 +++++++++++++++++++++++---------
 ooni/reporter.py                        |  19 +-
 ooni/templates/httpt.py                 |   2 +-
 ooni/utils/net.py                       |   4 +-
 setup.py                                |   5 +-
 10 files changed, 384 insertions(+), 157 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 16a6ba0..a06dc41 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -12,6 +12,8 @@ Vagrant.configure("2") do |config|
   # Place the ooni-backend source code in ../ooni-backend to sync it with the vagrant instance
   config.vm.synced_folder "../ooni-backend", "/backend"
 
+  config.vm.network :private_network, ip: "192.168.38.20"
+
 end
 
 $script = <<SCRIPT
diff --git a/bin/ooniprobequeue b/bin/ooniprobequeue
new file mode 100755
index 0000000..f81f156
--- /dev/null
+++ b/bin/ooniprobequeue
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+import sys
+import copy_reg
+from twisted.internet import reactor
+
+# This is a hack to overcome a bug in python
+from ooni.utils.hacks import patched_reduce_ex
+copy_reg._reduce_ex = patched_reduce_ex
+
+code=0
+
+from ooni.oonicli import runWithDaemonDirector
+d = runWithDaemonDirector()
+@xxxxxxxxx
+def cb(result):
+    global code
+    if result is not None:
+        code=1
+    reactor.stop()
+reactor.run()
+sys.exit(code)
diff --git a/ooni/director.py b/ooni/director.py
index 5ffee97..9bac49d 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -132,7 +132,7 @@ class Director(object):
                 yield config.check_tor()
             if config.advanced.start_tor:
                 yield self.startTor()
-            elif config.tor.control_port:
+            elif config.tor.control_port and config.tor_state is None:
                 log.msg("Connecting to Tor Control Port...")
                 yield self.getTorState()
 
diff --git a/ooni/geoip.py b/ooni/geoip.py
index 0ab2475..cb66675 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -190,34 +190,37 @@ class ProbeIP(object):
 
     @defer.inlineCallbacks
     def lookup(self):
-        try:
-            yield self.askTor()
-            log.msg("Found your IP via Tor %s" % self.address)
-            self.resolveGeodata()
+        if self.address:
             defer.returnValue(self.address)
-        except errors.TorStateNotFound:
-            log.debug("Tor is not running. Skipping IP lookup via Tor.")
-        except Exception:
-            log.msg("Unable to lookup the probe IP via Tor.")
-
-        try:
-            yield self.askTraceroute()
-            log.msg("Found your IP via Traceroute %s" % self.address)
-            self.resolveGeodata()
-            defer.returnValue(self.address)
-        except errors.InsufficientPrivileges:
-            log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient privileges")
-        except:
-            log.msg("Unable to lookup the probe IP via traceroute")
-
-        try:
-            yield self.askGeoIPService()
-            log.msg("Found your IP via a GeoIP service: %s" % self.address)
-            self.resolveGeodata()
-            defer.returnValue(self.address)
-        except Exception:
-            log.msg("Unable to lookup the probe IP via GeoIPService")
-            raise
+        else:
+            try:
+                yield self.askTor()
+                log.msg("Found your IP via Tor %s" % self.address)
+                self.resolveGeodata()
+                defer.returnValue(self.address)
+            except errors.TorStateNotFound:
+                log.debug("Tor is not running. Skipping IP lookup via Tor.")
+            except Exception:
+                log.msg("Unable to lookup the probe IP via Tor.")
+
+            try:
+                yield self.askTraceroute()
+                log.msg("Found your IP via Traceroute %s" % self.address)
+                self.resolveGeodata()
+                defer.returnValue(self.address)
+            except errors.InsufficientPrivileges:
+                log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges")
+            except:
+                log.msg("Unable to lookup the probe IP via traceroute")
+
+            try:
+                yield self.askGeoIPService()
+                log.msg("Found your IP via a GeoIP service: %s" % self.address)
+                self.resolveGeodata()
+                defer.returnValue(self.address)
+            except Exception:
+                log.msg("Unable to lookup the probe IP via GeoIPService")
+                raise
 
     @defer.inlineCallbacks
     def askGeoIPService(self):
diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py
index 997fb4e..f21b041 100644
--- a/ooni/nettests/blocking/http_requests.py
+++ b/ooni/nettests/blocking/http_requests.py
@@ -17,6 +17,9 @@ class UsageOptions(usage.Options):
         ['url', 'u', None, 'Specify a single URL to test.'],
         ['factor', 'f', 0.8,
          'What factor should be used for triggering censorship (0.8 == 80%)']]
+    optFlags = [
+        ["withoutbody","B", "don't include HTTP response body inside of the report"],
+        ]
 
 
 class HTTPRequestsTest(httpt.HTTPTest):
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 030014e..47aacfb 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -1,10 +1,14 @@
 import sys
+
 import os
+import json
 import yaml
+import random
+import urlparse
 
 from twisted.python import usage
 from twisted.python.util import spewer
-from twisted.internet import defer
+from twisted.internet import defer, reactor, protocol, task
 
 from ooni import errors, __version__
 
@@ -17,6 +21,9 @@ from ooni.utils import log
 from ooni.utils.net import hasRawSocketPermission
 
 
+
+class LifetimeExceeded(Exception): pass
+
 class Options(usage.Options):
     synopsis = """%s [options] [path to test].py
     """ % (os.path.basename(sys.argv[0]),)
@@ -49,7 +56,8 @@ class Options(usage.Options):
         ["datadir", "d", None,
          "Specify a path to the ooniprobe data directory"],
         ["annotations", "a", None,
-         "Annotate the report with a key:value[, key:value] format."]]
+         "Annotate the report with a key:value[, key:value] format."],
+        ["queue", "Q", None, "AMQP Queue URL amqp://user:pass@host:port/vhost/queue"]]
 
     compData = usage.Completions(
         extraActions=[usage.CompleteFiles(
@@ -102,11 +110,63 @@ def parseOptions():
     return dict(cmd_line_options)
 
 
-def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
-    """
-    Instance the director, parse command line options and start an ooniprobe
-    test!
-    """
+def director_startup_handled_failures(failure):
+    log.err("Could not start the director")
+    failure.trap(errors.TorNotRunning,
+                 errors.InvalidOONIBCollectorAddress,
+                 errors.UnableToLoadDeckInput,
+                 errors.CouldNotFindTestHelper,
+                 errors.CouldNotFindTestCollector,
+                 errors.ProbeIPUnknown,
+                 errors.InvalidInputFile,
+                 errors.ConfigFileIncoherent)
+
+    if isinstance(failure.value, errors.TorNotRunning):
+        log.err("Tor does not appear to be running")
+        log.err("Reporting with the collector %s is not possible" %
+                global_options['collector'])
+        log.msg(
+            "Try with a different collector or disable collector reporting with -n")
+
+    elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
+        log.err("Invalid format for oonib collector address.")
+        log.msg(
+            "Should be in the format http://<collector_address>:<port>")
+        log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
+
+    elif isinstance(failure.value, errors.UnableToLoadDeckInput):
+        log.err("Unable to fetch the required inputs for the test deck.")
+        log.msg(
+            "Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues";)
+
+    elif isinstance(failure.value, errors.CouldNotFindTestHelper):
+        log.err("Unable to obtain the required test helpers.")
+        log.msg(
+            "Try with a different bouncer or check that Tor is running properly.")
+
+    elif isinstance(failure.value, errors.CouldNotFindTestCollector):
+        log.err("Could not find a valid collector.")
+        log.msg(
+            "Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.")
+
+    elif isinstance(failure.value, errors.ProbeIPUnknown):
+        log.err("Failed to lookup probe IP address.")
+        log.msg("Check your internet connection.")
+
+    elif isinstance(failure.value, errors.InvalidInputFile):
+        log.err("Invalid input file \"%s\"" % failure.value)
+
+    elif isinstance(failure.value, errors.ConfigFileIncoherent):
+        log.err("Incoherent config file")
+
+    if config.advanced.debug:
+        log.exception(failure)
+
+def director_startup_other_failures(failure):
+    log.err("An unhandled exception occurred while starting the director!")
+    log.exception(failure)
+
+def setupGlobalOptions(logging, start_tor, check_incoherences):
     global_options = parseOptions()
     config.global_options = global_options
     config.set_paths()
@@ -133,49 +193,44 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
             log.err("Insufficient Privileges to capture packets."
                     " See ooniprobe.conf privacy.includepcap")
             sys.exit(2)
-
-    director = Director()
-    if global_options['list']:
-        print "# Installed nettests"
-        for net_test_id, net_test in director.getNetTests().items():
-            print "* %s (%s/%s)" % (net_test['name'],
-                                    net_test['category'],
-                                    net_test['id'])
-            print "  %s" % net_test['description']
-
-        sys.exit(0)
-
-    elif global_options['printdeck']:
-        del global_options['printdeck']
-        print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
-        print yaml.safe_dump([{'options': global_options}]).strip()
-
-        sys.exit(0)
-
-    if global_options.get('annotations') is not None:
-        annotations = {}
-        for annotation in global_options["annotations"].split(","):
-            pair = annotation.split(":")
-            if len(pair) == 2:
-                key = pair[0].strip()
-                value = pair[1].strip()
-                annotations[key] = value
-            else:
-                log.err("Invalid annotation: %s" % annotation)
-                sys.exit(1)
-        global_options["annotations"] = annotations
-
-    if global_options['no-collector']:
-        log.msg("Not reporting using a collector")
-        global_options['collector'] = None
-        start_tor = False
-    else:
-        start_tor = True
+    return global_options
+
+def setupAnnotations(global_options):
+    annotations={}
+    for annotation in global_options["annotations"].split(","):
+        pair = annotation.split(":")
+        if len(pair) == 2:
+            key = pair[0].strip()
+            value = pair[1].strip()
+            annotations[key] = value
+        else:
+            log.err("Invalid annotation: %s" % annotation)
+            sys.exit(1)
+    global_options["annotations"] = annotations
+    return annotations
+
+def setupCollector(global_options, net_test_loader):
+    collector = None
+    if not global_options['no-collector']:
+        if global_options['collector']:
+            collector = global_options['collector']
+        elif 'collector' in config.reports \
+                and config.reports['collector']:
+            collector = config.reports['collector']
+        elif net_test_loader.collector:
+            collector = net_test_loader.collector
+
+    if collector and collector.startswith('httpo:') \
+            and (not (config.tor_state or config.tor.socks_port)):
+        raise errors.TorNotRunning
+    return collector
+
+
+def createDeck(global_options,url=None,filename=None):
+    log.msg("Creating deck for: %s" %(url or filename,) )
 
     deck = Deck(no_collector=global_options['no-collector'])
     deck.bouncer = global_options['bouncer']
-    if global_options['collector']:
-        start_tor |= True
 
     try:
         if global_options['testdeck']:
@@ -183,7 +238,15 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
         else:
             log.debug("No test deck detected")
             test_file = nettest_to_path(global_options['test_file'], True)
-            net_test_loader = NetTestLoader(global_options['subargs'],
+            if url is not None:
+                args = ('-u',url)
+            elif filename is not None:
+                args = ('-f',filename)
+            else:
+                args = tuple()
+            if any(global_options['subargs']):
+                args = global_options['subargs'] + args
+            net_test_loader = NetTestLoader(args,
                                             test_file=test_file)
             if global_options['collector']:
                 net_test_loader.collector = global_options['collector']
@@ -205,6 +268,48 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
             log.exception(e)
         log.err(e)
         sys.exit(5)
+    return deck
+
+def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
+    """
+    Instance the director, parse command line options and start an ooniprobe
+    test!
+    """
+
+    global_options = setupGlobalOptions(logging, start_tor, check_incoherences)
+
+    director = Director()
+    if global_options['list']:
+        print "# Installed nettests"
+        for net_test_id, net_test in director.getNetTests().items():
+            print "* %s (%s/%s)" % (net_test['name'],
+                                    net_test['category'],
+                                    net_test['id'])
+            print "  %s" % net_test['description']
+
+        sys.exit(0)
+
+    elif global_options['printdeck']:
+        del global_options['printdeck']
+        print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
+        print yaml.safe_dump([{'options': global_options}]).strip()
+
+        sys.exit(0)
+
+    if global_options.get('annotations') is not None:
+        annotations = setupAnnotations(global_options)
+
+    if global_options['no-collector']:
+        log.msg("Not reporting using a collector")
+        global_options['collector'] = None
+        start_tor = False
+    else:
+        start_tor = True
+
+    if global_options['collector']:
+        start_tor |= True
+
+    deck = createDeck(global_options)
 
     start_tor |= deck.requiresTor
     d = director.start(start_tor=start_tor,
@@ -216,61 +321,6 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
         except errors.UnableToLoadDeckInput as error:
             return defer.failure.Failure(error)
 
-    def director_startup_handled_failures(failure):
-        log.err("Could not start the director")
-        failure.trap(errors.TorNotRunning,
-                     errors.InvalidOONIBCollectorAddress,
-                     errors.UnableToLoadDeckInput,
-                     errors.CouldNotFindTestHelper,
-                     errors.CouldNotFindTestCollector,
-                     errors.ProbeIPUnknown,
-                     errors.InvalidInputFile,
-                     errors.ConfigFileIncoherent)
-
-        if isinstance(failure.value, errors.TorNotRunning):
-            log.err("Tor does not appear to be running")
-            log.err("Reporting with the collector %s is not possible" %
-                    global_options['collector'])
-            log.msg(
-                "Try with a different collector or disable collector reporting with -n")
-
-        elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
-            log.err("Invalid format for oonib collector address.")
-            log.msg(
-                "Should be in the format http://<collector_address>:<port>")
-            log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
-
-        elif isinstance(failure.value, errors.UnableToLoadDeckInput):
-            log.err("Unable to fetch the required inputs for the test deck.")
-            log.msg(
-                "Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues";)
-
-        elif isinstance(failure.value, errors.CouldNotFindTestHelper):
-            log.err("Unable to obtain the required test helpers.")
-            log.msg(
-                "Try with a different bouncer or check that Tor is running properly.")
-
-        elif isinstance(failure.value, errors.CouldNotFindTestCollector):
-            log.err("Could not find a valid collector.")
-            log.msg(
-                "Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.")
-
-        elif isinstance(failure.value, errors.ProbeIPUnknown):
-            log.err("Failed to lookup probe IP address.")
-            log.msg("Check your internet connection.")
-
-        elif isinstance(failure.value, errors.InvalidInputFile):
-            log.err("Invalid input file \"%s\"" % failure.value)
-
-        elif isinstance(failure.value, errors.ConfigFileIncoherent):
-            log.err("Incoherent config file")
-
-        if config.advanced.debug:
-            log.exception(failure)
-
-    def director_startup_other_failures(failure):
-        log.err("An unhandled exception occurred while starting the director!")
-        log.exception(failure)
 
     # Wait until director has started up (including bootstrapping Tor)
     # before adding tests
@@ -285,19 +335,7 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
             # deck is a singleton, the default collector set in
             # ooniprobe.conf will be used
 
-            collector = None
-            if not global_options['no-collector']:
-                if global_options['collector']:
-                    collector = global_options['collector']
-                elif 'collector' in config.reports \
-                        and config.reports['collector']:
-                    collector = config.reports['collector']
-                elif net_test_loader.collector:
-                    collector = net_test_loader.collector
-
-            if collector and collector.startswith('httpo:') \
-                    and (not (config.tor_state or config.tor.socks_port)):
-                raise errors.TorNotRunning
+            collector = setupCollector(global_options, net_test_loader)
 
             test_details = net_test_loader.testDetails
             test_details['annotations'] = global_options['annotations']
@@ -307,11 +345,163 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
                                   collector)
         return director.allTestsDone
 
-    def start():
+
+    d.addCallback(setup_nettest)
+    d.addCallback(post_director_start)
+    d.addErrback(director_startup_handled_failures)
+    d.addErrback(director_startup_other_failures)
+    return d
+
+    
+
+# this variant version of runWithDirector splits the process in two,
+# allowing a single director instance to be reused with multiple decks.
+
+def runWithDaemonDirector(logging=True, start_tor=True, check_incoherences=True):
+    """
+    Instance the director, parse command line options and start an ooniprobe
+    test!
+    """
+
+    try:
+        import pika
+        from pika import exceptions
+        from pika.adapters import twisted_connection
+    except ImportError:
+        print "Pika is required for queue connection."
+        print "Install with \"pip install pika\"."
+        sys.exit(7)
+
+
+
+    global_options = setupGlobalOptions(logging, start_tor, check_incoherences)
+
+    director = Director()
+
+    if global_options.get('annotations') is not None:
+        annotations = setupAnnotations(global_options)
+
+    if global_options['no-collector']:
+        log.msg("Not reporting using a collector")
+        global_options['collector'] = None
+        start_tor = False
+    else:
+        start_tor = True
+
+
+    def run_test(global_options, url=None, filename=None):
+        assert url is not None or filename is not None
+
+        deck = createDeck(global_options, url=url, filename=filename)
+
+        d = director.start(start_tor=True,
+                           check_incoherences=check_incoherences)
+
+        def setup_nettest(_):
+            try:
+                return deck.setup()
+            except errors.UnableToLoadDeckInput as error:
+                return defer.failure.Failure(error)
+
+
+
+        # Wait until director has started up (including bootstrapping Tor)
+        # before adding tests
+        def post_director_start(_):
+            for net_test_loader in deck.netTestLoaders:
+                # Decks can specify different collectors
+                # for each net test, so that each NetTest
+                # may be paired with a test_helper and its collector
+                # However, a user can override this behavior by
+                # specifying a collector from the command-line (-c).
+                # If a collector is not specified in the deck, or the
+                # deck is a singleton, the default collector set in
+                # ooniprobe.conf will be used
+
+                collector = setupCollector(global_options, net_test_loader)
+
+                test_details = net_test_loader.testDetails
+                test_details['annotations'] = global_options['annotations']
+
+                director.startNetTest(net_test_loader,
+                                      global_options['reportfile'],
+                                      collector)
+            return director.allTestsDone
+ 
         d.addCallback(setup_nettest)
         d.addCallback(post_director_start)
         d.addErrback(director_startup_handled_failures)
         d.addErrback(director_startup_other_failures)
         return d
 
-    return start()
+    finished = defer.Deferred()
+
+    @defer.inlineCallbacks
+    def readmsg(_,channel, queue_object, consumer_tag, counter):
+
+        # Wait for a message and decode it.
+        if counter >= lifetime:
+            queue_object.close(LifetimeExceeded())
+            yield channel.basic_cancel(consumer_tag=consumer_tag)
+            finished.callback(None)
+
+        else:
+            log.msg("Waiting for message")
+            
+            try:
+                ch, method, properties, body = yield queue_object.get()
+                log.msg("Got message")
+                data = json.loads(body)
+                counter += 1
+
+                log.msg("Received %d/%d: %s" %(counter, lifetime, data['url'],))
+                # acknowledge the message
+                ch.basic_ack(delivery_tag=method.delivery_tag)
+
+                d = run_test(global_options, url=data['url'].encode('utf8'))
+                # When the test has been completed, go back to waiting for a message.
+                d.addCallback(readmsg, channel, queue_object, consumer_tag, counter+1)
+            except exceptions.AMQPError,v:
+                finished.errback(v)
+
+
+
+    @defer.inlineCallbacks
+    def runQueue(connection, name, qos):
+        # Set up the queue consumer.  When a message is received, run readmsg
+        channel = yield connection.channel()
+        yield channel.basic_qos(prefetch_count=qos)
+        queue_object, consumer_tag = yield channel.basic_consume(
+                                                   queue=name,
+                                                   no_ack=False)
+        readmsg(None,channel,queue_object,consumer_tag, 0)
+
+
+
+    # Create the AMQP connection.  This could be refactored to allow test URLs
+    # to be submitted through an HTTP server interface or something.
+    urlp = urlparse.urlparse(config.global_options['queue'])
+    urlargs = dict(urlparse.parse_qsl(urlp.query))
+
+    # random lifetime requests counter
+    lifetime = random.randint(820,1032)
+
+    # AMQP connection details are sent through the cmdline parameter '-Q'
+    
+    creds = pika.PlainCredentials(urlp.username or 'guest',
+                                  urlp.password or 'guest')
+    parameters = pika.ConnectionParameters(urlp.hostname,
+                                           urlp.port or 5672,
+                                           urlp.path.rsplit('/',1)[0] or '/',
+                                           creds,
+                                           heartbeat_interval=120,
+                                           )
+    cc = protocol.ClientCreator(reactor,
+                                twisted_connection.TwistedProtocolConnection,
+                                parameters)
+    d = cc.connectTCP(urlp.hostname, urlp.port or 5672)
+    d.addCallback(lambda protocol: protocol.ready)
+    # start the wait/process sequence.
+    d.addCallback(runQueue, urlp.path.rsplit('/',1)[-1], int(urlargs.get('qos',1)))
+
+    return finished
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 36a0e88..5e702bb 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -240,7 +240,7 @@ class OONIBReporter(OReporter):
         Will raise :class:ooni.errors.InvalidOONIBCollectorAddress an exception
         if the oonib reporter is not valid.
         """
-        regexp = '^(http|httpo):\/\/[a-zA-Z0-9\-\.]+(:\d+)?$'
+        regexp = '^(http|https|httpo):\/\/[a-zA-Z0-9\-\.]+(:\d+)?$'
         if not re.match(regexp, self.collectorAddress):
             raise errors.InvalidOONIBCollectorAddress
 
@@ -265,7 +265,7 @@ class OONIBReporter(OReporter):
         request_json = json.dumps(request)
         log.debug("Sending %s" % request_json)
 
-        bodyProducer = StringProducer(json.dumps(request))
+        bodyProducer = StringProducer(request_json)
 
         try:
             yield self.agent.request("PUT", url,
@@ -288,6 +288,7 @@ class OONIBReporter(OReporter):
         # tor is started.
 
         from ooni.utils.hacks import SOCKS5Agent
+        from twisted.web.client import Agent
         from twisted.internet import reactor
 
         if self.collectorAddress.startswith('httpo://'):
@@ -298,8 +299,13 @@ class OONIBReporter(OReporter):
             self.agent = SOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)
 
         elif self.collectorAddress.startswith('https://'):
-            # XXX add support for securely reporting to HTTPS collectors.
-            log.err("HTTPS based collectors are currently not supported.")
+            # not sure if there's something else it needs.  Seems to work.
+            # Very difficult to get it to work with self-signed certs.
+            self.agent = Agent(reactor)
+
+        elif self.collectorAddress.startswith('http://'):
+            log.msg("Warning using unencrypted collector")
+            self.agent = Agent(reactor)
 
         url = self.collectorAddress + '/report'
 
@@ -319,12 +325,15 @@ class OONIBReporter(OReporter):
             # the backend.
             'content': content
         }
+        # import values from the environment
+        request.update([(k.lower(),v) for (k,v) in os.environ.iteritems()
+                        if k.startswith('PROBE_')])
 
         log.msg("Reporting %s" % url)
         request_json = json.dumps(request)
         log.debug("Sending %s" % request_json)
 
-        bodyProducer = StringProducer(json.dumps(request))
+        bodyProducer = StringProducer(request_json)
 
         log.msg("Creating report with OONIB Reporter. Please be patient.")
         log.msg("This may take up to 1-2 minutes...")
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index e77913d..a753955 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -136,7 +136,7 @@ class HTTPTest(NetTestCase):
         if response:
             request_response['response'] = {
                 'headers': list(response.headers.getAllRawHeaders()),
-                'body': response_body,
+                'body': response_body if self.localOptions.get('withoutbody',1) == 0 else '',
                 'code': response.code
         }
         if failure_string:
diff --git a/ooni/utils/net.py b/ooni/utils/net.py
index 014020c..96d62a1 100644
--- a/ooni/utils/net.py
+++ b/ooni/utils/net.py
@@ -27,8 +27,6 @@ except ImportError:
 # agents with largest anonymity set.
 
 userAgents = ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7",
-              "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3 1 2 like Mac OS X; en-us)"
-              "AppleWebKit/528.18 (KHTML, like Gecko) Mobile/7D11",
               "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
               "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
               "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
@@ -169,4 +167,4 @@ def hasRawSocketPermission():
         socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_RAW)
         return True
     except socket.error:
-        return False
\ No newline at end of file
+        return False
diff --git a/setup.py b/setup.py
index 9b874f3..d99a90f 100644
--- a/setup.py
+++ b/setup.py
@@ -156,7 +156,7 @@ class install(_st_install):
         prefix = os.path.abspath(self.prefix)
         self.set_data_files(prefix)
         self.do_egg_install()
-        self.ooniresources()
+        #self.ooniresources()
 
 install_requires = []
 dependency_links = []
@@ -204,7 +204,8 @@ setup(
     packages=packages,
     include_package_data=True,
     scripts=["bin/oonideckgen", "bin/ooniprobe",
-             "bin/oonireport", "bin/ooniresources"],
+             "bin/oonireport", "bin/ooniresources",
+			 "bin/ooniprobequeue"],
     dependency_links=dependency_links,
     install_requires=install_requires,
     cmdclass={"install": install},



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits