[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Big refactor and code reorganization.



commit 4258d6e33547ab24d604d6e800dc1ee23df43b64
Author: Arturo Filastò <hellais@xxxxxxxxx>
Date:   Sun Mar 4 22:53:20 2012 -0800

    Big refactor and code reorganization.
---
 node.py             |   66 ----------
 ooniprobe.py        |    3 +-
 plugoo.py           |  348 ---------------------------------------------------
 plugoo/__init__.py  |   53 ++++++++
 plugoo/assets.py    |   45 +++++++
 plugoo/nodes.py     |   71 +++++++++++
 plugoo/reports.py   |  170 +++++++++++++++++++++++++
 plugoo/tests.py     |  105 +++++++++++++++
 tests/bridget.py    |   10 +-
 tests/dnstamper.py  |    8 +-
 tests/traceroute.py |    7 +-
 11 files changed, 460 insertions(+), 426 deletions(-)

diff --git a/node.py b/node.py
deleted file mode 100644
index eea78df..0000000
--- a/node.py
+++ /dev/null
@@ -1,66 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8
-
-import os
-import sys
-import socks
-
-class Node(object):
-    def __init__(self, address, port):
-        self.address = address
-        self.port = port
-
-"""
-[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5")
-[]: node_socket = node.wrap_socket()
-"""
-class NetworkNode(Node):
-    def __init__(self, address, port, node_type="SOCKS5", auth_creds=None):
-        self.node = Node(address,port)
-
-        # XXX support for multiple types
-        # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...)
-        self.node_type = node_type
-        # type-specific authentication credentials
-        self.auth_creds = auth_creds
-
-    def _get_socksipy_socket(self, proxy_type, auth_creds):
-        import socks
-        s = socks.socksocket()
-        # auth_creds[0] -> username
-        # auth_creds[1] -> password
-        s.setproxy(proxy_type, self.node.address, self.node.port,
-                   self.auth_creds[0], self.auth_creds[1])
-        return s
-
-    def _get_socket_wrapper(self):
-        if (self.node_type.startswith("SOCKS")): # SOCKS proxies
-            if (self.node_type != "SOCKS5"):
-                proxy_type = socks.PROXY_TYPE_SOCKS5
-            elif (self.node_type != "SOCKS4"):
-                proxy_type = socks.PROXY_TYPE_SOCKS4
-            else:
-                print "We don't know this proxy type."
-                sys.exit(1)
-
-            return self._get_socksipy_socket(proxy_type)
-        elif (self.node_type == "HTTP"): # HTTP proxies
-            return self._get_socksipy_socket(PROXY_TYPE_HTTP)
-        else: # Unknown proxies
-            print "We don't know this proxy type."
-            sys.exit(1)
-
-    def wrap_socket(self):
-        return self._get_socket_wrapper()
-
-class CodeExecNode(Node):
-    def __init__(self, address, port, node_type, auth_creds):
-        self.node = Node(address,port)
-
-        # node type (SSH proxy, etc.)
-        self.node_type = node_type
-        # type-specific authentication credentials
-        self.auth_creds = auth_creds
-
-    def add_unit(self):
-        pass
diff --git a/ooniprobe.py b/ooniprobe.py
index d8501c1..a542153 100755
--- a/ooniprobe.py
+++ b/ooniprobe.py
@@ -96,7 +96,8 @@ class ooni(object):
             test_name = fname
             if not self.config.main.testdir in sys.path:
                 sys.path.insert(0, self.config.main.testdir)
-
+            #print "Fname: %s\n__import__(%s)" % (fname, fname)
+            #print sys.path
             module = __import__(fname)
             try:
                 test.name = module.__plugoo__
diff --git a/plugoo.py b/plugoo.py
deleted file mode 100644
index 0ce1d28..0000000
--- a/plugoo.py
+++ /dev/null
@@ -1,348 +0,0 @@
-# -*- coding: UTF-8
-"""
-    plugoo
-    ******
-
-    This contains all of the "goo" necessary for creating
-    ooni-probe plugoonies.
-
-    :copyright: (c) 2012 by Arturo Filastò.
-    :license: see LICENSE for more details.
-
-"""
-
-import os
-from datetime import datetime
-import yaml
-
-try:
-    import socks
-except:
-    "Error SocksiPy is not installed!"
-import socket
-
-import logging
-import itertools
-import gevent
-
-class Asset:
-    """This is an ooni-probe asset. It is a python
-    iterator object, allowing it to be efficiently looped.
-    To create your own custom asset your should subclass this
-    and override the next_asset method and the len method for
-    computing the length of the asset.
-    """
-    def __init__(self, file=None):
-        self.fh = None
-        if file:
-            self.name = file
-            self.fh = open(file, 'r')
-        self.eof = False
-
-    def __iter__(self):
-        return self
-
-    def len(self):
-        """Returns the length of the asset
-        """
-        for i, l in enumerate(self.fh):
-            pass
-        # rewind the file
-        self.fh.seek(0)
-        return i + 1
-
-    def next_asset(self):
-        """Return the next asset.
-        """
-        # XXX this is really written with my feet.
-        #     clean me up please...
-        line = self.fh.readline()
-        if line:
-            return line.replace('\n','')
-        else:
-            self.fh.seek(0)
-            raise StopIteration
-
-    def next(self):
-        try:
-            return self.next_asset()
-        except:
-            raise StopIteration
-
-
-class Report:
-    """This is the ooni-probe reporting mechanism. It allows
-    reporting to multiple destinations and file formats.
-
-    :scp the string of <host>:<port> of an ssh server
-
-    :yaml the filename of a the yaml file to write
-
-    :file the filename of a simple txt file to write
-
-    :tcp the <host>:<port> of a TCP server that will just listen for
-         inbound connection and accept a stream of data (think of it
-         as a `nc -l -p <port> > filename.txt`)
-    """
-    def __init__(self, ooni,
-                 scp="127.0.0.1:22",
-                 file="test.report",
-                 tcp="127.0.0.1:9000"):
-
-        self.file = file
-        self.tcp = tcp
-        self.scp = scp
-        self.config = ooni.config.report
-        self.logger = ooni.logger
-
-        if self.config.timestamp:
-            tmp = self.file.split('.')
-            self.file = '.'.join(tmp[:-1]) + "-" + \
-                        datetime.now().isoformat('-') + '.' + \
-                        tmp[-1]
-            print self.file
-
-        try:
-            import paramiko
-        except:
-            self.scp = None
-            self.logger.warn("Could not import paramiko. SCP will not be disabled")
-
-    def __call__(self, data):
-        """
-        This should be invoked every time you wish to write some
-        data to the reporting system
-        """
-        #print "Writing report(s)"
-        dump = '--- \n'
-        dump += yaml.dump(data)
-        reports = []
-
-        if self.file:
-            reports.append("file")
-
-        if self.tcp:
-            reports.append("tcp")
-
-        if self.scp:
-            reports.append("scp")
-
-        jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports]
-        gevent.joinall(jobs)
-        ret = []
-        for job in jobs:
-            #print job.value
-            ret.append(job.value)
-        return ret
-
-    def file_report(self, data, file=None, mode='a+'):
-        """
-        This reports to a file in YAML format
-        """
-        if not file:
-            file = self.file
-        try:
-            f = open(file, mode)
-            f.write(data)
-        except Exception, e:
-            raise e
-        finally:
-            f.close()
-
-
-    def tcp_report(self, data):
-        """This connect to the specified tcp server
-        and writes the data passed as argument.
-        """
-        host, port = self.tcp.split(":")
-        tcp = socket.getprotobyname('tcp')
-        send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, tcp)
-        try:
-            send_socket.connect((host, int(port)))
-            send_socket.send(data)
-
-        except Exception, e:
-            raise e
-
-        finally:
-            send_socket.close()
-
-
-    def scp_report(self, data, rfile=None, mode='a+'):
-        """Push data to the remote ssh server.
-        :rfile the remote filename to write
-        :data the raw data content that should be written
-        :mode in what mode the file should be created
-        """
-        if not rfile:
-            rfile = self.file
-        host, port = self.scp.split(":")
-        transport = paramiko.Transport((host, port))
-
-        # The remote path of the remote file to write
-        rfpath = os.path.join(self.config.ssh_rpath, rfile)
-
-        try:
-            username = self.config.ssh_username
-        except:
-            raise "No username provided"
-
-        # Load the local known host key file
-        transport.load_host_keys(os.path.expanduser("~/.ssh/known_hosts"))
-
-        # We prefer to use an ssh keyfile fo authentication
-        if self.config.ssh_keyfile:
-            keyfile = os.path.expanduser(self.config.ssh_keyfile)
-            key = paramiko.RSAKey.from_private_key_file(keylocfile)
-            try:
-                transport.connect(username=username, pkey=key)
-            except Exception, e:
-                raise e
-
-        # If not even a password is fine
-        elif self.config.ssh_password:
-            try:
-                transport.connect(username=username, password=self.config.ssh_password)
-            except Exception, e:
-                raise e
-
-        # ... but no authentication, that is madness!
-        else:
-            raise "No key or password provided for ssh"
-
-        sftp = paramiko.SFTPClient.from_transport(transport)
-        try:
-            sftp = ssh.open_sftp()
-            remote_file = sftp.file(rfile, mode)
-            remote_file.set_pipelined(True)
-            remote_file.write(data)
-
-        except Exception, e:
-            raise e
-        sftp.close()
-        transport.close()
-
-
-    def send_report(self, data, type):
-        """This sends the report using the
-        specified type.
-        """
-        #print "Reporting %s to %s" % (data, type)
-        self.logger.info("Reporting to %s" % type)
-        getattr(self, type+"_report").__call__(data)
-
-class Plugoo():
-    def __init__(self, ooni):
-        self.config = ooni.config
-        self.logger = ooni.logger
-        self.name = "test"
-        self.report = Report(ooni,
-                             scp=ooni.config.report.ssh,
-                             file=ooni.config.report.file,
-                             tcp=ooni.config.report.tcp)
-
-
-    def control(self, *a, **b):
-        pass
-
-    def experiment(self, *a, **b):
-        """Override this method to write your own
-        Plugoo.
-        """
-        pass
-
-    def load_assets(self, assets):
-        """Takes as input an array of Asset objects and
-        outputs an iterator for the loaded assets.
-        example:
-        assets = [hostlist, portlist, requestlist]
-
-        """
-        asset_count = len(assets)
-        bigsize = 0
-        bigidx = 0
-
-        if asset_count > 1:
-            # If we have more than on asset we try to do some
-            # optimizations as how to iterate through them by
-            # picking the largest asset set as the main iterator
-            # and do a cartesian product on the smaller sets
-            for i, v in enumerate(assets):
-                size = v.len()
-                if size > bigsize:
-                    bigidx, bigsize = (i, size)
-
-            smallassets = list(assets)
-            smallassets.pop(bigidx)
-
-        for x in assets[bigidx]:
-            if asset_count > 1:
-                # XXX this will only work in python 2.6, maybe refactor?
-                for comb in itertools.product(*smallassets):
-                    yield (x,) + comb
-            else:
-                yield (x)
-
-    def srun(self, assets=None, buffer=10, timeout=2):
-        self.logger.info("Starting %s", self.name)
-        if assets:
-            self.logger.debug("Running through tests")
-            for i, data in enumerate(self.load_assets(assets)):
-                args = {'data': data}
-                ret = self.experiment(**args)
-                print ret
-                self.report(ret)
-
-    def run(self, assets=None, buffer=10, timeout=100000):
-        self.logger.info("Starting %s", self.name)
-        jobs = []
-        if assets:
-            self.logger.debug("Running through tests")
-            for i, data in enumerate(self.load_assets(assets)):
-                args = {'data': data}
-                # Append to the job queue
-                jobs.append(gevent.spawn(self.experiment, **args))
-                # If the buffer is full run the jobs
-                if i % buffer == (buffer-1):
-                    # Run the jobs with the selected timeout
-                    gevent.joinall(jobs, timeout=timeout)
-                    for job in jobs:
-                        #print "JOB VAL: %s" % job.value
-                        self.logger.info("Writing report(s)")
-                        self.report(job.value)
-                        job.kill()
-                    jobs = []
-
-            if len(jobs) > 0:
-                gevent.joinall(jobs, timeout=timeout)
-                for job in jobs:
-                    #print "JOB VAL: %s" % job.value
-                    self.logger.info("Writing report(s)")
-                    self.report(job.value)
-                    job.kill()
-                jobs = []
-
-class torify(object):
-    """This is the torify decorator. It should be used to
-    decorate functions that should use to for connecting to
-    the interwebz. The suggary syntax is the following:
-    @torify([urllib2])
-    def myfunction():
-        f = urllib2.urlopen('https://torproject.org/')
-    remember to set the proxyaddress in the config file.
-    """
-    def __init__(self, f):
-        print f
-        self.f = f
-
-    def __get__(self, instance, owner):
-        self.modules = instance.modules
-        def decorator(*args):
-            print instance.config.main.proxyaddress
-            host, port = instance.config.main.proxyaddress.split(":")
-            socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, int(port))
-            # Wrap the modules into socks
-            for module in self.modules:
-                socks.wrapmodule(module)
-            return self.f(instance, *args)
-        return decorator
diff --git a/plugoo/__init__.py b/plugoo/__init__.py
new file mode 100644
index 0000000..1bd4446
--- /dev/null
+++ b/plugoo/__init__.py
@@ -0,0 +1,53 @@
+# -*- coding: UTF-8
+"""
+    plugoo
+    ******
+
+    This contains all of the "goo" necessary for creating
+    ooni-probe plugoonies.
+
+    :copyright: (c) 2012 by Arturo Filastò.
+    :license: see LICENSE for more details.
+
+"""
+
+__all__ = ['assets', 'reports', 'nodes']
+
+import os
+from datetime import datetime
+import yaml
+
+try:
+    import socks
+except:
+    "Error SocksiPy is not installed!"
+import socket
+
+import logging
+import itertools
+import gevent
+
+class torify(object):
+    """This is the torify decorator. It should be used to
+    decorate functions that should use to for connecting to
+    the interwebz. The suggary syntax is the following:
+    @torify([urllib2])
+    def myfunction():
+        f = urllib2.urlopen('https://torproject.org/')
+    remember to set the proxyaddress in the config file.
+    """
+    def __init__(self, f):
+        print f
+        self.f = f
+
+    def __get__(self, instance, owner):
+        self.modules = instance.modules
+        def decorator(*args):
+            print instance.config.main.proxyaddress
+            host, port = instance.config.main.proxyaddress.split(":")
+            socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, host, int(port))
+            # Wrap the modules into socks
+            for module in self.modules:
+                socks.wrapmodule(module)
+            return self.f(instance, *args)
+        return decorator
diff --git a/plugoo/assets.py b/plugoo/assets.py
new file mode 100644
index 0000000..5f93d1b
--- /dev/null
+++ b/plugoo/assets.py
@@ -0,0 +1,45 @@
+class Asset:
+    """This is an ooni-probe asset. It is a python
+    iterator object, allowing it to be efficiently looped.
+    To create your own custom asset your should subclass this
+    and override the next_asset method and the len method for
+    computing the length of the asset.
+    """
+    def __init__(self, file=None, *args, **argv):
+        self.fh = None
+        if file:
+            self.name = file
+            self.fh = open(file, 'r')
+        self.eof = False
+
+    def __iter__(self):
+        return self
+
+    def len(self):
+        """Returns the length of the asset
+        """
+        for i, l in enumerate(self.fh):
+            pass
+        # rewind the file
+        self.fh.seek(0)
+        return i + 1
+
+    def next_asset(self):
+        """Return the next asset.
+        """
+        # XXX this is really written with my feet.
+        #     clean me up please...
+        line = self.fh.readline()
+        if line:
+            return line.replace('\n','')
+        else:
+            self.fh.seek(0)
+            raise StopIteration
+
+    def next(self):
+        try:
+            return self.next_asset()
+        except:
+            raise StopIteration
+
+
diff --git a/plugoo/nodes.py b/plugoo/nodes.py
new file mode 100644
index 0000000..a76601d
--- /dev/null
+++ b/plugoo/nodes.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8
+
+import os
+import sys
+import socks
+
+class Node(object):
+    def __init__(self, address, port):
+        self.address = address
+        self.port = port
+
+"""
+[]: node = NetworkNode("192.168.0.112", 5555, "SOCKS5")
+[]: node_socket = node.wrap_socket()
+"""
+class NetworkNode(Node):
+    def __init__(self, address, port, node_type="SOCKS5", auth_creds=None):
+        self.node = Node(address,port)
+
+        # XXX support for multiple types
+        # node type (SOCKS proxy, HTTP proxy, GRE tunnel, ...)
+        self.node_type = node_type
+        # type-specific authentication credentials
+        self.auth_creds = auth_creds
+
+    def _get_socksipy_socket(self, proxy_type, auth_creds):
+        import socks
+        s = socks.socksocket()
+        # auth_creds[0] -> username
+        # auth_creds[1] -> password
+        s.setproxy(proxy_type, self.node.address, self.node.port,
+                   self.auth_creds[0], self.auth_creds[1])
+        return s
+
+    def _get_socket_wrapper(self):
+        if (self.node_type.startswith("SOCKS")): # SOCKS proxies
+            if (self.node_type != "SOCKS5"):
+                proxy_type = socks.PROXY_TYPE_SOCKS5
+            elif (self.node_type != "SOCKS4"):
+                proxy_type = socks.PROXY_TYPE_SOCKS4
+            else:
+                print "We don't know this proxy type."
+                sys.exit(1)
+
+            return self._get_socksipy_socket(proxy_type)
+        elif (self.node_type == "HTTP"): # HTTP proxies
+            return self._get_socksipy_socket(PROXY_TYPE_HTTP)
+        else: # Unknown proxies
+            print "We don't know this proxy type."
+            sys.exit(1)
+
+    def wrap_socket(self):
+        return self._get_socket_wrapper()
+
+class CodeExecNode(Node):
+    def __init__(self, address, port, node_type, auth_creds):
+        self.node = Node(address,port)
+
+        # node type (SSH proxy, etc.)
+        self.node_type = node_type
+        # type-specific authentication credentials
+        self.auth_creds = auth_creds
+
+    def add_unit(self):
+        pass
+
+    def get_status(self):
+        pass
+
+
diff --git a/plugoo/reports.py b/plugoo/reports.py
new file mode 100644
index 0000000..5a9fb8b
--- /dev/null
+++ b/plugoo/reports.py
@@ -0,0 +1,170 @@
+import os
+from datetime import datetime
+import yaml
+
+import logging
+import itertools
+import gevent
+
+class Report:
+    """This is the ooni-probe reporting mechanism. It allows
+    reporting to multiple destinations and file formats.
+
+    :scp the string of <host>:<port> of an ssh server
+
+    :yaml the filename of a the yaml file to write
+
+    :file the filename of a simple txt file to write
+
+    :tcp the <host>:<port> of a TCP server that will just listen for
+         inbound connection and accept a stream of data (think of it
+         as a `nc -l -p <port> > filename.txt`)
+    """
+    def __init__(self, ooni,
+                 scp="127.0.0.1:22",
+                 file="test.report",
+                 tcp="127.0.0.1:9000"):
+
+        self.file = file
+        self.tcp = tcp
+        self.scp = scp
+        self.config = ooni.config.report
+        self.logger = ooni.logger
+
+        if self.config.timestamp:
+            tmp = self.file.split('.')
+            self.file = '.'.join(tmp[:-1]) + "-" + \
+                        datetime.now().isoformat('-') + '.' + \
+                        tmp[-1]
+            print self.file
+
+        try:
+            import paramiko
+        except:
+            self.scp = None
+            self.logger.warn("Could not import paramiko. SCP will not be disabled")
+
+    def __call__(self, data):
+        """
+        This should be invoked every time you wish to write some
+        data to the reporting system
+        """
+        #print "Writing report(s)"
+        dump = '--- \n'
+        dump += yaml.dump(data)
+        reports = []
+
+        if self.file:
+            reports.append("file")
+
+        if self.tcp:
+            reports.append("tcp")
+
+        if self.scp:
+            reports.append("scp")
+
+        jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports]
+        gevent.joinall(jobs)
+        ret = []
+        for job in jobs:
+            #print job.value
+            ret.append(job.value)
+        return ret
+
+    def file_report(self, data, file=None, mode='a+'):
+        """
+        This reports to a file in YAML format
+        """
+        if not file:
+            file = self.file
+        try:
+            f = open(file, mode)
+            f.write(data)
+        except Exception, e:
+            raise e
+        finally:
+            f.close()
+
+
+    def tcp_report(self, data):
+        """This connect to the specified tcp server
+        and writes the data passed as argument.
+        """
+        host, port = self.tcp.split(":")
+        tcp = socket.getprotobyname('tcp')
+        send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, tcp)
+        try:
+            send_socket.connect((host, int(port)))
+            send_socket.send(data)
+
+        except Exception, e:
+            raise e
+
+        finally:
+            send_socket.close()
+
+
+    def scp_report(self, data, rfile=None, mode='a+'):
+        """Push data to the remote ssh server.
+        :rfile the remote filename to write
+        :data the raw data content that should be written
+        :mode in what mode the file should be created
+        """
+        if not rfile:
+            rfile = self.file
+        host, port = self.scp.split(":")
+        transport = paramiko.Transport((host, port))
+
+        # The remote path of the remote file to write
+        rfpath = os.path.join(self.config.ssh_rpath, rfile)
+
+        try:
+            username = self.config.ssh_username
+        except:
+            raise "No username provided"
+
+        # Load the local known host key file
+        transport.load_host_keys(os.path.expanduser("~/.ssh/known_hosts"))
+
+        # We prefer to use an ssh keyfile fo authentication
+        if self.config.ssh_keyfile:
+            keyfile = os.path.expanduser(self.config.ssh_keyfile)
+            key = paramiko.RSAKey.from_private_key_file(keylocfile)
+            try:
+                transport.connect(username=username, pkey=key)
+            except Exception, e:
+                raise e
+
+        # If not even a password is fine
+        elif self.config.ssh_password:
+            try:
+                transport.connect(username=username, password=self.config.ssh_password)
+            except Exception, e:
+                raise e
+
+        # ... but no authentication, that is madness!
+        else:
+            raise "No key or password provided for ssh"
+
+        sftp = paramiko.SFTPClient.from_transport(transport)
+        try:
+            sftp = ssh.open_sftp()
+            remote_file = sftp.file(rfile, mode)
+            remote_file.set_pipelined(True)
+            remote_file.write(data)
+
+        except Exception, e:
+            raise e
+        sftp.close()
+        transport.close()
+
+
+    def send_report(self, data, type):
+        """This sends the report using the
+        specified type.
+        """
+        #print "Reporting %s to %s" % (data, type)
+        self.logger.info("Reporting to %s" % type)
+        getattr(self, type+"_report").__call__(data)
+
+
diff --git a/plugoo/tests.py b/plugoo/tests.py
new file mode 100644
index 0000000..24986b2
--- /dev/null
+++ b/plugoo/tests.py
@@ -0,0 +1,105 @@
+import os
+from datetime import datetime
+import yaml
+
+import logging
+import itertools
+import gevent
+from plugoo.reports import Report
+
+class Test:
+    """
+    This is a ooni probe Test.
+    Also known as a Plugoo!
+    """
+    def __init__(self, ooni):
+        self.config = ooni.config
+        self.logger = ooni.logger
+        self.name = "test"
+        self.report = Report(ooni,
+                             scp=ooni.config.report.ssh,
+                             file=ooni.config.report.file,
+                             tcp=ooni.config.report.tcp)
+
+
+    def control(self, *a, **b):
+        pass
+
+    def experiment(self, *a, **b):
+        """Override this method to write your own
+        Plugoo.
+        """
+        pass
+
+    def load_assets(self, assets):
+        """Takes as input an array of Asset objects and
+        outputs an iterator for the loaded assets.
+        example:
+        assets = [hostlist, portlist, requestlist]
+
+        """
+        asset_count = len(assets)
+        bigsize = 0
+        bigidx = 0
+
+        if asset_count > 1:
+            # If we have more than on asset we try to do some
+            # optimizations as how to iterate through them by
+            # picking the largest asset set as the main iterator
+            # and do a cartesian product on the smaller sets
+            for i, v in enumerate(assets):
+                size = v.len()
+                if size > bigsize:
+                    bigidx, bigsize = (i, size)
+
+            smallassets = list(assets)
+            smallassets.pop(bigidx)
+
+        for x in assets[bigidx]:
+            if asset_count > 1:
+                # XXX this will only work in python 2.6, maybe refactor?
+                for comb in itertools.product(*smallassets):
+                    yield (x,) + comb
+            else:
+                yield (x)
+
+    def srun(self, assets=None, buffer=10, timeout=2):
+        self.logger.info("Starting %s", self.name)
+        if assets:
+            self.logger.debug("Running through tests")
+            for i, data in enumerate(self.load_assets(assets)):
+                args = {'data': data}
+                ret = self.experiment(**args)
+                print ret
+                self.report(ret)
+
+    def run(self, assets=None, buffer=10, timeout=100000):
+        self.logger.info("Starting %s", self.name)
+        jobs = []
+        if assets:
+            self.logger.debug("Running through tests")
+            for i, data in enumerate(self.load_assets(assets)):
+                args = {'data': data}
+                # Append to the job queue
+                jobs.append(gevent.spawn(self.experiment, **args))
+                # If the buffer is full run the jobs
+                if i % buffer == (buffer-1):
+                    # Run the jobs with the selected timeout
+                    gevent.joinall(jobs, timeout=timeout)
+                    for job in jobs:
+                        #print "JOB VAL: %s" % job.value
+                        self.logger.info("Writing report(s)")
+                        self.report(job.value)
+                        job.kill()
+                    jobs = []
+
+            if len(jobs) > 0:
+                gevent.joinall(jobs, timeout=timeout)
+                for job in jobs:
+                    #print "JOB VAL: %s" % job.value
+                    self.logger.info("Writing report(s)")
+                    self.report(job.value)
+                    job.kill()
+                jobs = []
+
+
diff --git a/tests/bridget.py b/tests/bridget.py
index 0a2cc4f..61fc8b7 100644
--- a/tests/bridget.py
+++ b/tests/bridget.py
@@ -23,11 +23,11 @@ from subprocess import Popen, PIPE
 from datetime import datetime
 
 import shutil
-import plugoo
 import gevent
 from gevent import socket
 import fcntl
-from plugoo import Plugoo, Asset, torify
+from plugoo.assets import Asset
+from plugoo.tests import Test
 import urllib2
 import httplib
 import json
@@ -71,7 +71,7 @@ class BridgeTAsset(Asset):
     def __init__(self, file=None):
         self = Asset.__init__(self, file)
 
-class BridgeT(Plugoo):
+class BridgeT(Test):
     # This is the timeout value after which
     # we will give up
     timeout = 20
@@ -368,11 +368,11 @@ def run(ooni):
     bridges = BridgeTAsset(os.path.join(config.main.assetdir, \
                                         config.tests.tor_bridges))
 
-    assets = [bridges]
+    bridgelist = [bridges]
 
     bridget = BridgeT(ooni)
     ooni.logger.info("Starting bridget test")
-    bridget.run(assets)
+    bridget.run(bridgelist)
     bridget.print_failures()
     bridget.clean()
     ooni.logger.info("Testing completed!")
diff --git a/tests/dnstamper.py b/tests/dnstamper.py
index 06b3551..68be12d 100644
--- a/tests/dnstamper.py
+++ b/tests/dnstamper.py
@@ -5,16 +5,18 @@ except:
 import gevent
 import os
 import plugoo
-from plugoo import Plugoo, Asset
+from plugoo.assets import Asset
+from plugoo.tests import Test
+
 
 __plugoo__ = "DNST"
 __desc__ = "DNS censorship detection test"
 
 class DNSTAsset(Asset):
     def __init__(self, file=None):
-        self = Asset.__init__(self, file)
+        self = asset.__init__(self, file)
 
-class DNST(Plugoo):
+class DNST(Test):
     def lookup(self, hostname, ns):
         res = resolver.Resolver(configure=False)
         res.nameservers = [ns]
diff --git a/tests/traceroute.py b/tests/traceroute.py
index f2272c9..891d356 100644
--- a/tests/traceroute.py
+++ b/tests/traceroute.py
@@ -11,7 +11,8 @@ try:
 except:
     print "Error: traceroute plugin requires scapy to be installed (http://www.secdev.org/projects/scapy)"
 
-from plugoo import Plugoo, Asset
+from plugoo.assets import Asset
+from plugoo.tests import Test
 
 import socket
 
@@ -20,10 +21,10 @@ __desc__ = "Performs TTL walking tests"
 
 class TracerouteAsset(Asset):
     def __init__(self, file=None):
-        self = Asset.__init__(self, file)
+        self = asset.__init__(self, file)
 
 
-class Traceroute(Plugoo):
+class Traceroute(Test):
     """A *very* quick and dirty traceroute implementation, UDP and TCP
     """
     def traceroute(self, dst, dst_port=3880, src_port=3000, proto="tcp", max_hops=30):



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits