[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Get the new OONI architecture working properly.



commit c4c91d393ba3956b84d46521e2f58d30584b3ea0
Author: Arturo Filastò <hellais@xxxxxxxxxxxxxx>
Date:   Thu May 31 03:31:05 2012 +0200

    Get the new OONI architecture working properly.
    (kudos to the organizers of FreeBird :) )
---
 .gitignore                     |    3 +-
 ooni/oonicli.py                |   70 ++++++++----------------
 ooni/ooniprobe.log             |    3 +-
 ooni/plugins/.bridget.py.swp   |  Bin 12288 -> 0 bytes
 ooni/plugins/.dnstamper.py.swp |  Bin 12288 -> 0 bytes
 ooni/plugins/blocking.py       |   41 ++++++++++++++
 ooni/plugins/bridget.py        |   36 ------------
 ooni/plugins/dnstamper.py      |   22 --------
 ooni/plugins/dropin.cache      |   90 ++++++++++++++++++++++++++++++-
 ooni/plugins/skel.py           |    9 +++-
 ooni/plugoo/tests.py           |  118 ++++++++++++++--------------------------
 ooni/plugoo/work.py            |   64 +++++++++++++++-------
 ooni/skel.py                   |   10 ----
 oonib/oonibackend.py           |    8 ++--
 14 files changed, 255 insertions(+), 219 deletions(-)

diff --git a/.gitignore b/.gitignore
index 6433d66..8ed47e9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,5 @@ proxy-lists/ip-cc.txt
 proxy-lists/ips.txt
 proxy-lists/italy-dns-ips.txt
 proxy-lists/italy-http-ips.txt
-private/*
\ No newline at end of file
+private/*
+ooni/plugins/dropin.cache
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index d290e68..a0272fb 100755
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -9,7 +9,6 @@
 #
 
 import sys
-from plugoo import tests, work, assets
 
 from twisted.python import usage
 from twisted.plugin import getPlugins
@@ -19,8 +18,10 @@ from zope.interface.exceptions import BrokenImplementation
 from zope.interface.exceptions import BrokenMethodImplementation
 from zope.interface.verify import verifyObject
 from pprint import pprint
-from logo import getlogo
-import plugins
+
+from ooni.plugoo import tests, work, assets
+from ooni.logo import getlogo
+from ooni import plugins
 
 __version__ = "0.0.1-prealpha"
 
@@ -46,33 +47,20 @@ def retrieve_plugoo():
 
 plugoo = retrieve_plugoo()
 
-class StupidAsset(object):
-    def __init__(self):
-        self.idx = 0
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        if self.idx > 30:
-            raise StopIteration
-        self.idx += 1
-        return self.idx
-
-
 def runTest(test, options, global_options):
-    asset = None
-    if options['asset']:
-        print options['asset']
-        asset = assets.Asset(options['asset'])
-        print asset
 
-    wgen = work.WorkGenerator(asset, plugoo[test].__class__,
-            dict(options), start=options['resume'])
+    parallelism = int(global_options['parallelism'])
+    worker = work.Worker(parallelism)
+    test = plugoo[test].__class__
+
+    #if options['asset']:
+    #    print options['asset']
+    #    asset = assets.Asset(options['asset'])
+    #    print asset
 
-    if global_options['parallelism']:
-        wgen.size = int(global_options['parallelism'])
-        worker = work.Worker(wgen.size)
+    wgen = work.WorkGenerator(test(options, global_options),
+                              dict(options),
+                              start=options['resume'])
 
     for x in wgen:
         worker.push(x)
@@ -86,16 +74,17 @@ class Options(usage.Options):
         subCommands.append([test, None, plugoo[test].options, "Run the %s test" % test])
 
     optFlags = [
-        ['local', 'l', "If the test should be run locally (without having oonid running)"],
-        ['status', 'x', 'Show current state'],
-        ['restart', 'r', 'Restart OONI']
+        #['remote', 'r', "If the test should be run remotely (not supported)"],
+        #['status', 'x', 'Show current state'],
+        #['restart', 'r', 'Restart OONI']
     ]
 
     optParameters = [
         ['parallelism', 'n', 10, "Specify the number of parallel tests to run"],
-        ['target-node', 't', 'localhost:31415', 'Select target node'],
-        ['ooninet', 'o', 'localhost:4242', "Select OONI-net address for reporting"],
-        ['password', 'p', 'opennetwork', "Specify the password for authentication"],
+        #['target-node', 't', 'localhost:31415', 'Select target node'],
+        ['output', 'o', 'report.log', "Specify output report file"],
+        ['log', 'l', 'oonicli.log', "Specify output log file"],
+        #['password', 'p', 'opennetwork', "Specify the password for authentication"],
     ]
 
     def opt_version(self):
@@ -116,24 +105,11 @@ class Options(usage.Options):
 config = Options()
 config.parseOptions()
 
-if config['status']:
-    print "oonid is not running."
-    sys.exit(0)
-
-if config['restart']:
-    print "Restarting oonid."
-    sys.exit(0)
-
 if not config.subCommand:
     print "Error! No Test Specified."
     config.opt_help()
     sys.exit(1)
 
-if config['local']:
-    runTest(config.subCommand, config.subOptions, config)
+runTest(config.subCommand, config.subOptions, config)
 
-else:
-    print "This feature is currently not supported. :("
-    print "Use -l to run the test locally."
-    sys.exit(0)
 
diff --git a/ooni/ooniprobe.log b/ooni/ooniprobe.log
index d3914dc..700e820 100644
--- a/ooni/ooniprobe.log
+++ b/ooni/ooniprobe.log
@@ -1,2 +1 @@
-2012-05-29 01:32:22,811 ooniprobe    INFO     Started ooni-probe
-2012-05-29 01:32:22,972 ooniprobe    WARNING  Soft fail 'module' object has no attribute '__plugoo__'
+2012-05-31 02:27:28,010 ooniprobe    INFO     Started ooni-probe
diff --git a/ooni/plugins/.bridget.py.swp b/ooni/plugins/.bridget.py.swp
deleted file mode 100644
index 226d826..0000000
Binary files a/ooni/plugins/.bridget.py.swp and /dev/null differ
diff --git a/ooni/plugins/.dnstamper.py.swp b/ooni/plugins/.dnstamper.py.swp
deleted file mode 100644
index 9acb1c9..0000000
Binary files a/ooni/plugins/.dnstamper.py.swp and /dev/null differ
diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py
new file mode 100644
index 0000000..bbcb1a2
--- /dev/null
+++ b/ooni/plugins/blocking.py
@@ -0,0 +1,41 @@
+from zope.interface import implements
+from twisted.python import usage
+from twisted.plugin import IPlugin
+
+from ooni.plugoo.assets import Asset
+from ooni.plugoo.tests import ITest, TwistedTest
+
+class BlockingArgs(usage.Options):
+    optParameters = [['asset', 'a', None, 'Asset file'],
+                     ['resume', 'r', 0, 'Resume at this index'],
+                     ['other', 'o', None, 'Other arguments']]
+
+class BlockingTest(TwistedTest):
+    implements(IPlugin, ITest)
+
+    shortName = "blocking"
+    description = "Blocking plugin"
+    requirements = None
+    options = BlockingArgs
+    # Tells this to be blocking.
+    blocking = True
+
+    def control(self, experiment_result, args):
+        print "Experiment Result:", experiment_result
+        print "Args", args
+        return experiment_result
+
+    def experiment(self, args):
+        import urllib
+        req = urllib.urlopen(args['asset'])
+        return {'page': req.readlines()}
+
+    def load_assets(self):
+        if self.local_options:
+            return {'asset': Asset(self.local_options['asset'])}
+        else:
+            return {}
+
+# We need to instantiate it otherwise getPlugins does not detect it
+# XXX Find a way to load plugins without instantiating them.
+blocking = BlockingTest(None, None, None)
diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py
deleted file mode 100644
index 7271468..0000000
--- a/ooni/plugins/bridget.py
+++ /dev/null
@@ -1,36 +0,0 @@
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from plugoo.tests import ITest, TwistedTest
-from twisted.internet import threads
-
-from tests.bridget import BridgeT as BridgeTlegacy
-from tests.bridget import BridgeTAsset as BridgeTAsset
-from ooniprobe import ooni
-
-o = ooni()
-
-class BridgetArgs(usage.Options):
-    optParameters = [['asset', 'a', None, 'Asset file'],
-                     ['resume', 'r', 0, 'Resume at this index'],
-                     ['bridge', 'b', None, 'Specify a single bridge']]
-
-class BridgeT(TwistedTest):
-    implements(IPlugin, ITest)
-
-    shortName = "bridget"
-    description = "Bridget plugin"
-    requirements = None
-    options = BridgetArgs
-
-    def experiment(self):
-        bridget = BridgeTlegacy(o)
-        o.logger.info("Starting bridget test")
-        print "ASSET:%s " % self.asset
-        d = threads.deferToThread(bridget.connect, self.asset)
-        d.addCallback(self.d_experiment.callback, None)
-        return d
-
-# We need to instantiate it otherwise getPlugins does not detect it
-# XXX Find a way to load plugins without instantiating them.
-bridget = BridgeT(None, None)
diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py
deleted file mode 100644
index b8b7c7f..0000000
--- a/ooni/plugins/dnstamper.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from plugoo.tests import ITest, TwistedTest
-
-class SkelArgs(usage.Options):
-    optParameters = [['hosts', 'f', None, 'Hostname asset file'],
-                     ['dnsservers', 'd', None, 'DNS server asset file'],
-                     ['resume', 'r', 0, 'Resume at this index'],
-                     ['other', 'o', None, 'Other arguments']]
-
-class SkelTest(TwistedTest):
-    implements(IPlugin, ITest)
-
-    shortName = "skeleton"
-    description = "Skeleton plugin"
-    requirements = None
-    options = SkelArgs
-
-# We need to instantiate it otherwise getPlugins does not detect it
-# XXX Find a way to load plugins without instantiating them.
-skel = SkelTest(None, None)
diff --git a/ooni/plugins/dropin.cache b/ooni/plugins/dropin.cache
index 9258b18..bd26c09 100755
--- a/ooni/plugins/dropin.cache
+++ b/ooni/plugins/dropin.cache
@@ -53,7 +53,7 @@ g5
 NtRp24
 (dp25
 g8
-S'plugins.bridget'
+S'ooni.plugins.bridget'
 p26
 sg10
 Nsg11
@@ -73,4 +73,92 @@ sg21
 S'bridget'
 p31
 sg10
+NsbasbsS'blocking'
+p32
+g3
+(g4
+g5
+NtRp33
+(dp34
+g8
+S'ooni.plugins.blocking'
+p35
+sg10
+Nsg11
+(lp36
+g3
+(g13
+g5
+NtRp37
+(dp38
+g16
+(lp39
+g18
+acooni.plugoo.tests
+ITest
+p40
+asg20
+g33
+sg21
+S'blocking'
+p41
+sg10
+NsbasbsS'httphost'
+p42
+g3
+(g4
+g5
+NtRp43
+(dp44
+g8
+S'ooni.plugins.httphost'
+p45
+sg10
+S'\n    HTTP Host based filtering\n    *************************\n\n    This test detect HTTP Host field\n    based filtering.\n'
+p46
+sg11
+(lp47
+g3
+(g13
+g5
+NtRp48
+(dp49
+g16
+(lp50
+g18
+ag40
+asg20
+g43
+sg21
+S'httphost'
+p51
+sg10
+NsbasbsS'tcpscan'
+p52
+g3
+(g4
+g5
+NtRp53
+(dp54
+g8
+S'plugins.tcpscan'
+p55
+sg10
+Nsg11
+(lp56
+g3
+(g13
+g5
+NtRp57
+(dp58
+g16
+(lp59
+g18
+ag40
+asg20
+g53
+sg21
+S'tcpscan'
+p60
+sg10
 Nsbasbs.
\ No newline at end of file
diff --git a/ooni/plugins/skel.py b/ooni/plugins/skel.py
index 93bc1bb..00e4311 100644
--- a/ooni/plugins/skel.py
+++ b/ooni/plugins/skel.py
@@ -15,7 +15,14 @@ class SkelTest(TwistedTest):
     description = "Skeleton plugin"
     requirements = None
     options = SkelArgs
+    blocking = False
+
+    def load_assets(self):
+        if self.local_options:
+            return {'asset': open(self.local_options['asset'])}
+        else:
+            return {}
 
 # We need to instantiate it otherwise getPlugins does not detect it
 # XXX Find a way to load plugins without instantiating them.
-skel = SkelTest(None, None)
+skel = SkelTest(None, None, None)
diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py
index 09363a2..75d0765 100644
--- a/ooni/plugoo/tests.py
+++ b/ooni/plugoo/tests.py
@@ -6,9 +6,13 @@ from zope.interface import Interface, Attribute
 import logging
 import itertools
 import gevent
-from twisted.internet import reactor, defer
+
+from twisted.internet import reactor, defer, threads
 from twisted.python import failure
-from plugoo.reports import Report
+
+from ooni.plugoo import assets, work
+from ooni.plugoo.reports import Report
+
 
 class Test:
     """
@@ -128,102 +132,64 @@ class ITest(Interface):
     #node = Attribute("""This represents the node that will run the test""")
     options = Attribute("""These are the arguments to be passed to the test for it's execution""")
 
-    def startTest():
+    blocking = Attribute("""True or False, stating if the test should be run in a thread or not.""")
+
+    def startTest(asset):
         """
         Launches the Test with the specified arguments on a node.
         """
 
-#class HTTPRequestTest(HTTPClient):
-class HTTPRequestTest(object):
-    """
-    This is an example of how I would like to be able to write a test.
-
-    *BEWARE* this actually does not currently work, it's just an example of the
-    kind of API that I am attempting to achieve to simplify the writing of
-    tests.
+class TwistedTest(object):
+    blocking = False
 
-    implements(ITest)
+    def __init__(self, local_options, global_options, ooninet=None):
+        self.local_options = local_options
+        self.global_options = global_options
+        self.assets = self.load_assets()
+        #self.ooninet = ooninet
 
-    """
-    def startTest():
-        # The response object should also contain the request
+    def load_assets(self):
         """
-        response = {'response': {'headers': ..., 'content': ...,
-        'runtime': ..., 'timestamp': ...},
-        'request': {'headers': ..., 'content', 'timestamp', ...}
-        }
-        response = self.http_request(address, headers)
-        if response.headers['content'].matches("Some string"):
-            self.censorship = True
-            return response
-        else:
-            self.censorship = False
-            return response
-
+        This method should be overriden by the test writer to provide the logic
+        for loading their assets.
         """
-        pass
-
-class TwistedTest(object):
-    def __init__(self, asset, arguments, ooninet=None):
-        self.asset = asset
-        self.arguments = arguments
-        self.start_time = datetime.now()
-        self._parse_arguments()
-        #self.ooninet = ooninet
+        return {'asset': None}
 
     def __repr__(self):
-        return "<TwistedTest %s %s>" % (self.arguments, self.asset)
+        return "<TwistedTest %s %s %s>" % (self.options, self.global_options,
+                                           self.assets)
 
-    def _parse_arguments(self):
-        print self.arguments
-        if self.arguments and 'test' in self.arguments:
-            self.test = self.arguments['test']
-
-    def finished(self, result):
+    def finished(self, control):
         #self.ooninet.report(result)
-        print "FINIHSED"
         self.end_time = datetime.now()
+        result = {}
         result['start_time'] = self.start_time
         result['end_time'] = self.end_time
         result['run_time'] = self.end_time - self.start_time
+        result['control'] = control
+        print "FINISHED", result
+        return result
 
-    def _do_experiment(self):
-        self.d = defer.maybeDeferred(self.experiment)
-        self.d.addCallback(self.control)
+    def _do_experiment(self, args):
+        if self.blocking:
+            self.d = threads.deferToThread(self.experiment, args)
+        else:
+            self.d = defer.maybeDeferred(self.experiment, args)
+
+        self.d.addCallback(self.control, args)
         self.d.addCallback(self.finished)
         return self.d
 
-    def control(self, exp):
+    def control(self, result, args):
         print "Doing control..."
-        self.d.callback(result)
+        return result
 
-    def experiment(self):
+    def experiment(self, args):
         print "Doing experiment"
-        self.d_experiment.callback(None)
-
-    def startTest(self):
-        print "Starting test %s" % repr(self)
-        return self._do_experiment()
-
-class TwistedTestFactory(object):
+        return {}
 
-    test = None
-
-    def __init__(self, assets, node,
-                 idx=0):
-        """
-        """
-        self.assets = assets
-        self.node = node
-        self.idx = idx
-        self.workunit = WorkUnitFactory(assets)
-
-    def build_test(self):
-        """
-        Returns a TwistedTest instance
-        """
-        workunit = self.workunit.next()
-        t = self.test(node, workunit)
-        t.factory = self
-        return t
+    def startTest(self, args):
+        self.start_time = datetime.now()
+        print "Starting test %s" % self.__class__
+        return self._do_experiment(args)
 
diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py
index 7be4758..55bbd1d 100644
--- a/ooni/plugoo/work.py
+++ b/ooni/plugoo/work.py
@@ -10,15 +10,16 @@
     :license: see LICENSE for more details.
 
 """
-from datetime import datetime
+import itertools
 import yaml
+from datetime import datetime
 
 from zope.interface import Interface, Attribute
 
 from twisted.python import failure
 from twisted.internet import reactor, defer
 
-from plugoo.nodes import LocalNode
+from ooni.plugoo.nodes import LocalNode
 
 class Worker(object):
     """
@@ -34,9 +35,9 @@ class Worker(object):
         self._running -= 1
         if self._running < self.maxconcurrent and self._queued:
             workunit, d = self._queued.pop(0)
-            for work in workunit:
-                self._running += 1
-                actuald = work.startTest().addBoth(self._run)
+            asset, test, idx = workunit
+            self._running += 1
+            actuald = test.startTest(asset).addBoth(self._run)
         if isinstance(r, failure.Failure):
             r.trap()
 
@@ -44,14 +45,13 @@ class Worker(object):
         print r['start_time']
         print r['end_time']
         print r['run_time']
-        print repr(r)
         return r
 
     def push(self, workunit):
         if self._running < self.maxconcurrent:
-            for work in workunit:
-                self._running += 1
-                work.startTest().addBoth(self._run)
+            asset, test, idx = workunit
+            self._running += 1
+            test.startTest(asset).addBoth(self._run)
             return
         d = defer.Deferred()
         self._queued.append((workunit, d))
@@ -59,6 +59,8 @@ class Worker(object):
 
 class WorkUnit(object):
     """
+    XXX This is currently not implemented for KISS sake.
+
     This is an object responsible for completing WorkUnits it will
     return its result in a deferred.
 
@@ -77,21 +79,21 @@ class WorkUnit(object):
     test = None
     arguments = None
 
-    def __init__(self, asset, test, idx, arguments=None):
+    def __init__(self, asset, assetNames, test, idx):
         self.asset = asset
         if not asset:
             self.assetGenerator = iter([1])
         else:
             self.assetGenerator = iter(asset)
         self.Test = test
-        self.arguments = arguments
+        self.assetNames = assetNames
         self.idx = idx
 
     def __iter__(self):
         return self
 
     def __repr__(self):
-        return "<WorkUnit %s %s %s>" % (self.arguments, self.Test, self.idx)
+        return "<WorkUnit %s %s %s>" % (self.assetNames, self.Test, self.idx)
 
     def serialize(self):
         """
@@ -105,7 +107,7 @@ class WorkUnit(object):
         """
         try:
             asset = self.assetGenerator.next()
-            ret = self.Test(asset, self.arguments)
+            ret = self.Test.set_asset(asset)
             return ret
         except StopIteration:
             raise StopIteration
@@ -120,10 +122,12 @@ class WorkGenerator(object):
     """
     size = 10
 
-    def __init__(self, asset, test, arguments=None, start=None):
-        self.assetGenerator = asset
+    def __init__(self, test, arguments=None, start=None):
         self.Test = test
-        self.arguments = arguments
+
+        self.assetGenerator = itertools.product(*self.Test.assets.values())
+        self.assetNames = self.Test.assets.keys()
+
         self.idx = 0
         self.end = False
         if start:
@@ -139,12 +143,32 @@ class WorkGenerator(object):
             self.idx += 1
 
     def next(self):
+        if not self.assetGenerator:
+            self.end = True
+            return (self.assetNames, self.Test, self.idx)
+
+        try:
+            asset = self.assetGenerator.next()
+            ret = {}
+            for i, v in enumerate(asset):
+                ret[self.assetNames[i]] = v
+        except StopIteration:
+            raise StopIteration
+
+        self.idx += 1
+        print "IDX:%s" % self.idx
+        return (ret, self.Test, self.idx)
+
+    def p_next(self):
+        """
+        XXX This is not used for KISS sake.
+        """
         if self.end:
             raise StopIteration
 
         if not self.assetGenerator:
             self.end = True
-            return WorkUnit(None, self.Test, self.idx, self.arguments)
+            return WorkUnit(None, self.assetNames, self.Test, self.idx)
 
         # Plank asset
         p_asset = []
@@ -152,11 +176,13 @@ class WorkGenerator(object):
             try:
                 asset = self.assetGenerator.next()
                 p_asset.append(asset)
-                print asset
+                print p_asset
             except StopIteration:
+                if self.asset_num > 1:
+                    pass
                 self.end = True
                 break
 
         self.idx += 1
-        return WorkUnit(p_asset, self.Test, self.idx, self.arguments)
+        return WorkUnit(p_asset, self.assetNames, self.Test, self.idx)
 
diff --git a/ooni/skel.py b/ooni/skel.py
deleted file mode 100644
index 8982106..0000000
--- a/ooni/skel.py
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8
-"""
-    XXX
-    ***
-
-    :copyright: (c) 2012 by Arturo Filastò
-    :license: see LICENSE for more details.
-"""
-
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 774a9c5..fc5bbdb 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -16,19 +16,19 @@ from twisted.web import resource, server, static
 from twisted.web.microdom import escape
 from twisted.names import dns
 
-from backend.httpbackend import HTTPBackend
-from backend.dnsbackend import ProxyDNSServer
+from oonib.httpbackend import HTTPBackend
+from oonib.dnsbackend import ProxyDNSServer
 
 # This tells twisted to set the
 server.version = "Apache"
 
 application = service.Application('oonibackend')
 serviceCollection = service.IServiceCollection(application)
-internet.TCPServer(8000, server.Site(HTTPBackend())).setServiceParent(serviceCollection)
+internet.TCPServer(2000, server.Site(HTTPBackend())).setServiceParent(serviceCollection)
 
 # Start the DNS Server related services
 TCPDNSServer = ProxyDNSServer()
 internet.TCPServer(8002, TCPDNSServer).setServiceParent(serviceCollection)
 UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer)
-internet.UDPServer(5353, UDPFactory).setServiceParent(serviceCollection)
+internet.UDPServer(5354, UDPFactory).setServiceParent(serviceCollection)
 



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits