[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Port Bridget to the new API



commit de0c47f7152b3a0a9dd2baf27ce2b028819e3f63
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Mon Nov 5 20:23:27 2012 +0100

    Port Bridget to the new API
    * Unbreak the new API and make sure tests work as they should
    * Add more advanced parameters for the test input as requested by Isis
---
 nettests/bridget.py     |   89 +++--------
 ooni/__init__.py        |   18 +--
 ooni/inputunit.py       |  133 +---------------
 ooni/nettest.py         |  399 ++++++++---------------------------------------
 ooni/oonicli.py         |    2 +
 ooni/reporter.py        |   10 +-
 ooni/runner.py          |  237 +++++++++++-----------------
 ooni/templates/httpt.py |   19 +++
 ooni/utils/onion.py     |    4 +-
 9 files changed, 221 insertions(+), 690 deletions(-)

diff --git a/nettests/bridget.py b/nettests/bridget.py
index a334747..74d1d47 100644
--- a/nettests/bridget.py
+++ b/nettests/bridget.py
@@ -20,23 +20,25 @@ import os
 import sys
 
 from twisted.python       import usage
-from twisted.plugin       import IPlugin
 from twisted.internet     import defer, error, reactor
-from zope.interface       import implements
+
+from ooni                 import nettest
 
 from ooni.utils           import log, date
 from ooni.utils.config    import ValueChecker
 
-from ooni.plugoo.tests    import ITest, OONITest
-from ooni.plugoo.assets   import Asset, MissingAssetException
 from ooni.utils.onion     import TxtorconImportError
 from ooni.utils.onion     import PTNoBridgesException, PTNotFoundException
 
+
 try:
     from ooni.utils.onion     import parse_data_dir
 except:
     log.msg("Please go to /ooni/lib and do 'make txtorcon' to run this test!")
 
+class MissingAssetException(Exception):
+    pass
+
 class RandomPortException(Exception):
     """Raised when using a random port conflicts with configured ports."""
     def __init__(self):
@@ -83,18 +85,7 @@ class BridgetArgs(usage.Options):
         if self['torpath']:
             ValueChecker.file_check(self['torpath'])
 
-class BridgetAsset(Asset):
-    """Class for parsing bridget Assets ignoring commented out lines."""
-    def __init__(self, file=None):
-        self = Asset.__init__(self, file)
-
-    def parse_line(self, line):
-        if line.startswith('#'):
-            return
-        else:
-            return line.replace('\n','')
-
-class BridgetTest(OONITest):
+class BridgetTest(nettest.TestCase):
     """
     XXX fill me in
 
@@ -114,15 +105,13 @@ class BridgetTest(OONITest):
     :ivar tor_binary:
         Path to the Tor binary to use, e.g. \'/usr/sbin/tor\'
     """
-    implements(IPlugin, ITest)
+    name    = "bridget"
+    author  = "Isis Lovecruft <isis@xxxxxxxxxxxxxx>"
+    version = "0.1"
+    description   = "Use a Tor process to test connecting to bridges or relays"
+    advancedOptParameters = BridgetArgs
 
-    shortName    = "bridget"
-    description  = "Use a Tor process to test connecting to bridges or relays"
-    requirements = None
-    options      = BridgetArgs
-    blocking     = False
-
-    def initialize(self):
+    def setUp(self):
         """
         Extra initialization steps. We only want one child Tor process
         running, so we need to deal with most of the TorConfig() only once,
@@ -134,14 +123,16 @@ class BridgetTest(OONITest):
         self.tor_binary      = '/usr/sbin/tor'
         self.data_directory  = None
 
-        def __make_asset_list__(opt, lst):
+        def read_from_file(filename):
             log.msg("Loading information from %s ..." % opt)
-            with open(opt) as opt_file:
-                for line in opt_file.readlines():
+            with open(filename) as fp:
+                lst = []
+                for line in fp.readlines():
                     if line.startswith('#'):
                         continue
                     else:
                         lst.append(line.replace('\n',''))
+                return lst
 
         def __count_remaining__(which):
             total, reach, unreach = map(lambda x: which[x],
@@ -168,23 +159,24 @@ class BridgetTest(OONITest):
         self.relays['remaining']   = lambda: __count_remaining__(self.relays)
         self.relays['current']     = None
 
-        if self.local_options:
+        if self.localOptions:
             try:
-                from ooni.lib.txtorcon import TorConfig
+                from txtorcon import TorConfig
             except ImportError:
                 raise TxtorconImportError
             else:
                 self.config = TorConfig()
             finally:
-                options = self.local_options
+                options = self.localOptions
 
             if options['bridges']:
                 self.config.UseBridges = 1
-                __make_asset_list__(options['bridges'], self.bridges['all'])
+                self.bridges['all'] = read_from_file(options['bridges'])
             if options['relays']:
                 ## first hop must be in TorState().guards
+                # XXX where is this defined?
                 self.config.EntryNodes = ','.join(relay_list)
-                __make_asset_list__(options['relays'], self.relays['all'])
+                self.relays['all'] = read_from_file(options['relays'])
             if options['socks']:
                 self.socks_port = options['socks']
             if options['control']:
@@ -215,33 +207,7 @@ class BridgetTest(OONITest):
             self.config.ControlPort          = self.control_port
             self.config.CookieAuthentication = 1
 
-    def __load_assets__(self):
-        """
-        Load bridges and/or relays from files given in user options. Bridges
-        should be given in the form IP:ORport. We don't want to load these as
-        assets, because it's inefficient to start a Tor process for each one.
-
-        We cannot use the Asset model, because that model calls
-        self.experiment() with the current Assets, which would be one relay
-        and one bridge, then it gives the defer.Deferred returned from
-        self.experiment() to self.control(), which means that, for each
-        (bridge, relay) pair, experiment gets called again, which instantiates
-        an additional Tor process that attempts to bind to the same
-        ports. Thus, additionally instantiated Tor processes return with
-        RuntimeErrors, which break the final defer.chainDeferred.callback(),
-        sending it into the errback chain.
-        """
-        assets = {}
-        if self.local_options:
-            if self.local_options['bridges']:
-                assets.update({'bridge':
-                               BridgetAsset(self.local_options['bridges'])})
-            if self.local_options['relays']:
-                assets.update({'relay':
-                               BridgetAsset(self.local_options['relays'])})
-        return assets
-
-    def experiment(self, args):
+    def test_bridget(self):
         """
         if bridges:
             1. configure first bridge line
@@ -465,7 +431,7 @@ class BridgetTest(OONITest):
         #self.reactor.run()
         return state
 
-    def startTest(self, args):
+    def disabled_startTest(self, args):
         """
         Local override of :meth:`OONITest.startTest` to bypass calling
         self.control.
@@ -483,9 +449,6 @@ class BridgetTest(OONITest):
         self.d.addCallbacks(self.finished, log.err)
         return self.d
 
-## So that getPlugins() can register the Test:
-#bridget = BridgetTest(None, None, None)
-
 ## ISIS' NOTES
 ## -----------
 ## TODO:
diff --git a/ooni/__init__.py b/ooni/__init__.py
index 2f7c19f..f025d30 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,11 +1,3 @@
-__all__ = [
-    'config', 'inputunit', 'kit',
-    'lib', 'nettest', 'oonicli', 'reporter',
-    'runner', 'templates', 'utils',
-    ## XXX below are legacy related modules
-    'ooniprobe', 'plugoo', 'plugins'
-    ]
-
 from . import config
 from . import inputunit
 from . import kit
@@ -17,7 +9,13 @@ from . import runner
 from . import templates
 from . import utils
 
-## XXX below are legacy related modules
+# XXX below are legacy related modules
 #from . import ooniprobe
-from . import plugoo
+#from . import plugoo
 #from . import plugins
+
+__all__ = ['config', 'inputunit', 'kit',
+           'lib', 'nettest', 'oonicli', 'reporter',
+           'runner', 'templates', 'utils']
+           # XXX below are legacy related modules
+           #'ooniprobe', 'plugoo', 'plugins']
diff --git a/ooni/inputunit.py b/ooni/inputunit.py
index 054b3a9..e5b6187 100644
--- a/ooni/inputunit.py
+++ b/ooni/inputunit.py
@@ -1,31 +1,5 @@
-# -*- coding: utf-8 -*-
-#
-# inputunit.py
-# ------------
-# Classes and function for working with OONI TestCase inputs.
-#
-# @authors: Arturo Filasto, Isis Lovecruft
-# @version: 0.1.0-alpha
-# @license: see included LICENSE file
-# @copyright: 2012 Arturo Filasto, Isis Lovecruft, The Tor Project Inc.
-#
-
 from twisted.trial import unittest
 
-from zope.interface.exceptions import BrokenImplementation
-
-
-def simpleInputUnitProcessor(input_unit):
-    """A simple InputUnit generator without parsing."""
-    try:
-        assert hasattr(input_unit, '__iter__'), "inputs must be iterable!"
-    except AssertionError, ae:
-        raise BrokenImplementation, ae
-    else:
-        for i in input_unit:
-            yield i
-
-
 class PatchedPyUnitResultAdapter(unittest.PyUnitResultAdapter):
     def __init__(self, original):
         """
@@ -39,8 +13,8 @@ unittest.PyUnitResultAdapter = PatchedPyUnitResultAdapter
 
 class InputUnitFactory(object):
     """
-    This is a factory that takes the size of input units to be generated and a
-    set of units that is a python iterable item, and outputs InputUnit objects
+    This is a factory that takes the size of input units to be generated a set
+    of units that is a python iterable item and outputs InputUnit objects
     containing inputUnitSize elements.
 
     This object is a python iterable, this means that it does not need to keep
@@ -74,6 +48,7 @@ class InputUnitFactory(object):
 
         return InputUnit(input_unit_elements)
 
+
 class InputUnit(object):
     """
     This is a python iterable object that contains the input elements to be
@@ -101,107 +76,5 @@ class InputUnit(object):
     def append(self, input):
         self._inputs.append(input)
 
-class IUProcessorExit(GeneratorExit):
-    """InputUnitProcessor has exited."""
-
-class IUProcessorAborted(Exception):
-    """The InputUnitProcessor was aborted with Inputs remaining."""
-
-class InputUnitProcessor(InputUnit):
-    """
-    Create a generator for returning inputs one-by-one from a
-    :class:`InputUnit` (or any other iterable defined within an instance of
-    :class:`ooni.nettest.NetTestCase`), and a generator function.
-
-    The :ivar:generator can be a custom generator, or chain of generators, for
-    customized parsing of an InputUnit, or it can be an imported
-    function. There are useful imported functions in the builtin
-    :mod:`itertools`. If no :ivar:`generator` is given, the default one strips
-    whitespace characters, then returns the Input if the input does not begin
-    with a crunch (#) or bang (!). :)
-
-    If :ivar:catchStopIter is True, then catch any StopIterations and return a
-    2-tuple containing (boolean, list(unprocessed)):
-
-        (True, None) when :ivar:iterable is empty, or
-        (False, [unprocessed]) if :ivar:iterable was not empty.
-
-    If :ivar:catchStopIter is False (default), then we catch the StopIteration
-    exception, mark :attr:`empty` as 'True', and reraise the StopIteration.
 
-    XXX fill me in with parameter details
-    XXX I'm not sure if we need this class anymore
-    """
-    empty = False
-
-    def __init__(self, iterable, input_filter=None, catch_err=False):
-        """
-        Create an InputUnitProcessor.
-
-        XXX fill me in
-        """
-        from itertools import takewhile
-        from types     import GeneratorType
-
-        assert hasattr(iterable, "__iter__"), "That's not an iterable!"
-
-        self._iterable = iterable
-        self._infilter = input_filter
-        self._noerr    = catch_err
-        self._empty    = self.empty
 
-    def __len__(self):
-        return len(self.iterable)
-
-    def __unprocessed__(self):
-        if not self.isdone():
-            unprocessed = \
-                [u for u in self._unit if self._unit is not StopIteration]
-            return (False, unprocessed)
-        else:
-            return (True, None)
-
-    def isdone(self):
-        return self._empty
-
-    def throw(self, exception, message="", traceback=None):
-        try:
-            raise exception, message, traceback
-        except:
-            yield self._iterable.next()
-
-    @staticmethod
-    def __strip__(x):
-        return x.strip()
-
-    @staticmethod
-    def _default_input_filter(x):
-        if not x.startswith('#') and not x.startswith('!'):
-            return True
-        else:
-            return False
-
-    def __make_filter__(self):
-        if self.input_filter and hasattr(self.input_filter, "__call__"):
-            return self.input_filter
-        else:
-            return self._default_input_filter
-
-    def finish(self):
-        if self._noerr:
-            return self.__unprocessed__()
-        else:
-            if not self.isdone():
-                raise IUProcessorAborted
-            else:
-                raise IUProcessorExit
-
-    def process(self):
-        carbon = self.__make_filter__()
-        try:
-            yield takewhile(carbon,
-                            [i for i in self.__strip__(self._iterable)]
-                            ).next()
-        except StopIteration, si:
-            self._empty = True
-            yield self.finish()
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 94c8c56..c2a8575 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -1,17 +1,8 @@
 # -*- encoding: utf-8 -*-
 #
-#     nettest.py
-# ------------------->
-#
-# :authors: Arturo "hellais" Filastò <art@xxxxxxxxx>,
-#           Isis Lovecruft <isis@xxxxxxxxxxxxxx>
+# :authors: Arturo "hellais" Filastò <art@xxxxxxxxx>
 # :licence: see LICENSE
-# :copyright: 2012 Arturo Filasto, Isis Lovecruft
-# :version: 0.1.0-alpha
-#
-# <-------------------
 
-from functools import partial
 import sys
 import os
 import itertools
@@ -25,7 +16,6 @@ from ooni.utils import log
 
 pyunit = __import__('unittest')
 
-
 class InputTestSuite(pyunit.TestSuite):
     """
     This in an extension of a unittest test suite. It adds support for inputs
@@ -66,306 +56,26 @@ class InputTestSuite(pyunit.TestSuite):
             self._idx += 1
         return result
 
-class NetTestAdaptor(unittest.TestCase):
-    """
-    XXX fill me in
-    """
-
-    # @classmethod
-    # def __new__(cls, *args, **kwargs):
-    #     try:
-    #         setUpClass()
-    #     except Exception, e:
-    #         log.debug("NetTestAdaptor: constructor could not find setUpClass")
-    #         log.err(e)
-    #     return super( NetTestAdaptor, cls ).__new__(cls, *args, **kwargs)
-
-    def __init__(self, *args, **kwargs):
-        """
-        If you override me, you must call
-
-            ``super(NetTestCase, self).__init__(*args, **kwargs)``
-
-        at the beginning of your __init__ method. Keyword arguments passed to
-        the above statement become attributes of the adaptor, and can be used
-        to alter the logic of input handling and parent class instantiation.
-        Therefore, You probably do not need to pass me any keyword arguments
-        when calling me, i.e. using ``(*args, **kwargs)`` will work just fine.
-        """
-        log.debug("NetTestAdaptor: created")
-        if kwargs:
-            if 'methodName' in kwargs:
-                log.debug("NetTestAdaptor: found 'methodName' in kwargs")
-                log.debug("NetTestAdaptor: calling unittest.TestCase.__init()")
-                super( NetTestAdaptor, self ).__init__(
-                    methodName=kwargs['methodName'] )
-            else:
-                log.debug("NetTestAdaptor: calling unittest.TestCase.__init()")
-                super( NetTestAdaptor, self ).__init__( )
-
-            for key, value in kwargs.items():     ## Let subclasses define their
-                if key != 'methodName':           ## instantiation without
-                    if not hasattr(self, key):    ## overriding parent classes
-                        log.debug("NetTestAdaptor: calling setattr(self,%s,%s)"
-                                  % (key, value) )
-                        setattr(self, key, value)
-
-        #setattr(self, "copyattr", __copy_attr__)
-
-        ## Internal attribute copies:
-        #self._input_parser = copyattr("inputParser", alt=__input_parser__)
-        #self._nettest_name = copyattr("name", alt="NetTestAdaptor"))
-
-        #self.setUpClass(self.__class__)
 
-        #if hasattr(self, parsed_inputs):
-        #    self.inputs = self.parsed_inputs
-        #else:
-        #    log.debug("Unable to find parsed inputs")
-
-    @staticmethod
-    def __copyattr__(obj, old, new=None, alt=None):
-        """
-        Assign me to a new attribute name to have a copy of the old
-        attribute, if it exists.
-
-        Example:
-        >>> self.sun = "black"
-        >>> self._watermelon = __copyattr__(self, "sun")
-        >>> self._clocknoise = __copyattr__(self, "sound")
-        >>> print self._watermelon
-            black
-        >>> print self._clocknoise
-
-        :param old:
-            A string representing the name of the old attribute
-        :param new:
-            (Optional) A string to set as the new attribute name.
-        :param alt:
-            (Optional) An alternate value to return if the old
-            attribute is not found.
-        :return:
-            If :param:`old` is found, I return it's value.
-
-            If :param:`old` is not found:
-                If :param:`alt` is given, I return :param:`alt`.
-                Else, I return None.
-
-            If :param:`new` is set, I do not return anything, and
-            instead I set the new attribute's name to :param:`name`
-            and it's value to the value which I would have otherwise
-            returned.
-        """
-        if not new:
-            if not alt:
-                if hasattr(obj, old):
-                    return getattr(obj, old)
-                return
-            else:
-                if hasattr(obj, old):
-                    return getattr(obj, old)
-                return alt
-        else:
-            if not alt:
-                if hasattr(obj, old):
-                    _copy = getattr(obj, old)
-                else:
-                    copy = None
-                setattr(obj, new, _copy)
-            else:
-                if hasattr(obj, old):
-                    _copy = getattr(obj, old)
-                else:
-                    _copy = alt
-                setattr(obj, new, _copy)
-
-    def copyattr(self, *args, **kwargs):
-        if len(args) >= 1:
-           _copy = partial(__copyattr__, args[0])
-           if len(args) == 2:
-               return _copy(new=args[1])
-           elif len(args) == 3:
-               return _copy(new=args[1], alt=args[2])
-           elif kwargs:
-               return _copy(kwargs)
-        else:
-           return
-
-    @staticmethod
-    def __input_parser__(one_input): return one_input
-
-    @classmethod
-    def __get_inputs__(cls):
-        """
-        I am called during class setup and you probably should not override
-        me. I gather the internal inputs from :class:`NetTestCase` attributes
-        and pass them through :meth:`NetTestCase.inputParser`.  If you are
-        looking for a way to parse inputs from inputFile, see
-        :meth:`inputParser`. If :class:`NetTestCase` has an attribute
-        :attr:`inputFile`, I also handle opening that file, striping each line
-        of whitespace, and then sending the line to
-        :meth:`NetTestCase.inputParser`.
-
-        All inputs which I find, both statically set inputs and those returned
-        from processing an inputFile, I add to a list :ivar:`parsed`, after
-        parsing them. I return :ivar:`parsed`:
-
-        :ivar parsed:
-            A list of parsed inputs.
-        :return:
-            :ivar:`parsed`.
-        """
-        parsed = []
-
-        if cls._raw_inputs:
-            if isinstance(cls._raw_inputs, (list, tuple,)):
-                if len(cls._raw_inputs) > 0:
-                    if len(cls._raw_inputs) == 1 and cls._raw_inputs[0] is None:
-                        pass       ## don't burn cycles on testing null inputs
-                    else:
-                        log.msg("Received direct inputs:\n%s" % cls._raw_inputs)
-                        parsed.extend(
-                            [cls._input_parser(x) for x in cls._raw_inputs])
-            elif isinstance(cls._raw_inputs, str):
-                separated = cls._raw_inputs.translate(None, ',') ## space delineates
-                inputlist = separated.split(' ')
-                parsed.extend([cls._input_parser(x) for x in inputlist])
-            else:
-                log.debug("inputs not string or list; type: %s"
-                          % type(cls._raw_inputs))
-
-        if cls.subarg_inputs:
-            log.debug("NetTestAdaptor: __get_inputs__ found subarg_inputs=%s"
-                      % cls.subarg_inputs)
-            parsed.extend([cls._input_parser(x) for x in cls.subarg_inputs])
-
-        if cls._input_file:
-            try:
-                log.debug("NetTestAdaptor: __get_inputs__ Opening input file")
-                fp = open(cls._input_file)
-            except:
-                log.debug("NetTestAdaptor: __get_inputs__ Couldn't open input file")
-            else:
-                log.debug("NetTestAdaptor: __get_inputs__ Running input file processor")
-                lines = [line.strip() for line in fp.readlines()]
-                fp.close()
-
-                ## add to what we've already parsed, if any:
-                log.debug("NetTestAdaptor: __get_inputs__ Parsing lines from input file")
-                parsed.extend([cls._input_parser(ln) for ln in lines])
-        else:
-            log.debug("NetTestAdaptor: %s specified that it doesn't need inputFile."
-                      % cls._nettest_name)
-
-        return parsed
-
-    @classmethod
-    def __getopt__(cls, parseArgs=None):
-        """
-        Constuctor for a custom t.p.usage.Options class, per NetTestCase.
-
-        old code from runner.py:
-            opts = Options()
-            opts.parseOptions(config['subArgs'])
-            cls.localOptions = opts
-        """
-        if cls._testopt_params or cls._input_file:
-            if not cls._testopt_params:
-                cls._testopt_params = []
-
-            if cls._input_file:
-                cls._testopt_params.append(cls.input_file)
-
-        class NetTestOptions(usage.Options):
-            """Per NetTestCase Options class."""
-            optParameters     = cls._testopt_params
-            optFlags          = cls._testopt_flags
-            subOptions        = cls._sub_options
-            subCommands       = cls._sub_commands
-            defaultSubCommand = cls._default_subcmd
-            ## XXX i'm not sure if this part will work:
-            parseArgs         = lambda a: cls.subarg_inputs.append(a)
-
-            def opt_version(self):
-                """Display test version and exit."""
-                print "Test version: ", cls._nettest_version
-                sys.exit(0)
-
-        options = NetTestOptions()
-        return options
-
-        #if cls._input_file:
-        #    cls._input_file = cls.options[cls._input_file[0]]
-
-    @classmethod
-    def addSubArgToInputs(cls, subarg):
-        cls.subarg_inputs.append(subarg)
-
-    @classmethod
-    def buildOptions(cls, from_global):
-        log.debug("NetTestAdaptor: getTestOptions called")
-        options = cls.__getopt__()
-        log.debug("NetTestAdaptor: getTestOptions: cls.options = %s"
-                  % options)
-        options.parseOptions(from_global)
-        setattr(cls, "local_options", options)
-        log.debug("NetTestAdaptor: getTestOptions: cls.local_options = %s"
-                  % cls.local_options)
-
-    @classmethod
-    def setUpClass(cls):
-        """
-        Create a NetTestCase. To add futher setup steps before a set of tests
-        in a TestCase instance run, create a function called 'setUp'.
-
-        Class attributes, such as `report`, `optParameters`, `name`, and
-        `author` should be overriden statically as class attributes in any
-        subclass of :class:`ooni.nettest.NetTestCase`, so that the calling
-        functions during NetTestCase class setup can handle them correctly.
-        """
-
-        log.debug("NetTestAdaptor: setUpClass called")
-
-        ## These internal inputs are for handling inputs and inputFile
-        cls._raw_inputs   = cls.__copyattr__(cls, "inputs")
-        cls._input_file   = cls.__copyattr__(cls, "inputFile")
-        cls._input_parser = cls.__copyattr__(cls, "inputParser",
-                                             alt=cls.__input_parser__)
-        cls._nettest_name = cls.__copyattr__(cls, "name", alt="NetTestAdaptor")
-
-        ## This creates a class attribute with all of the parsed inputs,
-        ## which the instance will later set to be `self.inputs`.
-        cls.parsed_inputs = cls.__get_inputs__()
-        cls.subarg_inputs = cls.__copyattr__(cls, "subarg_inputs",
-                                             alt=[])
-
-        ## XXX we should handle options generation here
-        cls._testopt_params  = cls.__copyattr__(cls, "optParameters")
-        cls._testopt_flags   = cls.__copyattr__(cls, "optFlags")
-        cls._sub_options     = cls.__copyattr__(cls, "subOptions")
-        cls._sub_commands    = cls.__copyattr__(cls, "subCommands")
-        cls._default_subcmd  = cls.__copyattr__(cls, "defaultSubCommand")
-        cls._nettest_version = cls.__copyattr__(cls, "version")
-
-class NetTestCase(NetTestAdaptor):
+class NetTestCase(unittest.TestCase):
     """
     This is the monad of the OONI nettest universe. When you write a nettest
     you will subclass this object.
 
     * inputs: can be set to a static set of inputs. All the tests (the methods
-      starting with the "test_" prefix) will be run once per input.  At every
-      run the _input_ attribute of the TestCase instance will be set to the
-      value of the current iteration over inputs.  Any python iterable object
-      can be set to inputs.
+      starting with the "test_" prefix) will be run once per input.  At every run
+      the _input_ attribute of the TestCase instance will be set to the value of
+      the current iteration over inputs.  Any python iterable object can be set
+      to inputs.
 
-    * inputFile: attribute should be set to an array containing the command
-      line argument that should be used as the input file. Such array looks
-      like this:
+    * inputFile: attribute should be set to an array containing the command line
+      argument that should be used as the input file. Such array looks like
+      this:
 
-          ``["commandlinearg", "c", "The description"]``
+          ``["commandlinearg", "c", "default value" "The description"]``
 
-      The second value of such arrray is the shorthand for the command line
-      arg.  The user will then be able to specify inputs to the test via:
+      The second value of such arrray is the shorthand for the command line arg.
+      The user will then be able to specify inputs to the test via:
 
           ``ooniprobe mytest.py --commandlinearg path/to/file.txt``
 
@@ -374,9 +84,9 @@ class NetTestCase(NetTestAdaptor):
           ``ooniprobe mytest.py -c path/to/file.txt``
 
 
-    * inputParser: should be set to a function that takes as argument an
-      iterable containing test inputs and it simply parses those inputs and
-      returns them back to test instance.
+    * inputProcessor: should be set to a function that takes as argument an
+      open file descriptor and it will return the input to be passed to the test
+      instance.
 
     * name: should be set to the name of the test.
 
@@ -388,26 +98,44 @@ class NetTestCase(NetTestAdaptor):
     * version: is the version string of the test.
 
     * requiresRoot: set to True if the test must be run as root.
+
+    * optFlags: is assigned a list of lists. Each list represents a flag parameter, as so:
+
+        optFlags = [['verbose', 'v', 'Makes it tell you what it doing.'], | ['quiet', 'q', 'Be vewy vewy quiet.']]
+
+    As you can see, the first item is the long option name (prefixed with
+    '--' on the command line), followed by the short option name (prefixed with
+    '-'), and the description. The description is used for the built-in handling of
+    the --help switch, which prints a usage summary.
+
+
+    * optParameters: is much the same, except the list also contains a default value:
+
+        | optParameters = [['outfile', 'O', 'outfile.log', 'Description...']]
+
+    * advancedOptParameters: a subclass of twisted.python.usage.Options for more advanced command line arguments fun.
+
     """
     name = "I Did Not Change The Name"
     author = "Jane Doe <foo@xxxxxxxxxxx>"
-    version = "0.0.0"
+    version = "0"
 
+    inputs = [None]
     inputFile = None
-    inputs    = [None]
 
     report = {}
     report['errors'] = []
 
-    optParameters = None
     optFlags = None
-    subCommands = None
+    optParameters = None
+    advancedOptParameters = None
+
     requiresRoot = False
 
     def deferSetUp(self, ignored, result):
         """
-        If we have the reporterFactory set we need to write the header. If
-        such method is not present we will only run the test skipping header
+        If we have the reporterFactory set we need to write the header. If such
+        method is not present we will only run the test skipping header
         writing.
         """
         if result.reporterFactory.firstrun:
@@ -420,35 +148,34 @@ class NetTestCase(NetTestAdaptor):
             log.debug("Not first run. Running test setup directly")
             return unittest.TestCase.deferSetUp(self, ignored, result)
 
-    def inputParser(self, inputs):
-        """Replace me with a custom function for parsing inputs."""
-        log.debug("Running custom input processor")
-        return inputs
+    def inputProcessor(self, fp):
+        log.debug("Running default input processor")
+        for x in fp.readlines():
+            yield x.strip()
+        fp.close()
 
     def getOptions(self):
         log.debug("Getting options for test")
-
-        if self.local_options:
-            log.debug("NetTestCase: getOptions: self.local_options=%s"
-                      % str(self.local_options))
-        else:
-            log.debug("could not find cls.localOptions!")
-
-        return {'inputs': self.parsed_inputs,
+        if self.inputFile:
+            try:
+                assert isinstance(self.inputFile, str)
+            except AssertionError, ae:
+                log.err(ae)
+            else:
+                if os.path.isfile(self.inputFile):
+                    print self.inputFile
+                    fp = open(self.inputFile)
+                    self.inputs = self.inputProcessor(fp)
+        elif not self.inputs[0]:
+            pass
+        elif self.inputFile:
+            raise usage.UsageError("No input file specified!")
+        # XXX perhaps we may want to name and version to be inside of a
+        # different object that is not called options.
+        return {'inputs': self.inputs,
                 'name': self.name,
                 'version': self.version}
-        # if options:
-        #     return options
-        # else:
-        #     ## is this safe to do? it might turn Hofstaeder...
-        #     return self.__dict__
-        ####################
-        ## original return
-        ####################
-        #return {'inputs': self.inputs,
-        #        'name': self.name,
-        #        'version': self.version}
-    '''
+
     def __repr__(self):
         return "<%s inputs=%s>" % (self.__class__, self.inputs)
-    '''
+
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index b9f1dee..aa3cdc0 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -88,6 +88,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):
 
 def run():
     log.start()
+    log.debug("Started logging")
 
     if len(sys.argv) == 1:
         sys.argv.append("--help")
@@ -106,3 +107,4 @@ def run():
     for idx, cases in enumerate(casesList):
         orunner = runner.ORunner(cases, options[idx], config)
         orunner.run()
+
diff --git a/ooni/reporter.py b/ooni/reporter.py
index b856948..14327ef 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -16,7 +16,7 @@ from twisted.python.util import untilConcludes
 from twisted.trial import reporter
 from twisted.internet import defer
 
-from ooni.templates.httpt import BodyReceiver
+from ooni.templates.httpt import BodyReceiver, StringProducer
 from ooni.utils import date, log, geodata
 
 try:
@@ -178,9 +178,9 @@ class ReporterFactory(OReporter):
     @defer.inlineCallbacks
     def writeHeader(self):
         self.firstrun = False
-        (klass, options) = self.options
+        options = self.options
         self._writeln("###########################################")
-        self._writeln("# OONI Probe Report for %s test" % klass.name)
+        self._writeln("# OONI Probe Report for %s test" % options['name'])
         self._writeln("# %s" % date.pretty_date())
         self._writeln("###########################################")
 
@@ -207,8 +207,8 @@ class ReporterFactory(OReporter):
                         'probeLocation': {'city': client_geodata['city'],
                                           'countrycode':
                                           client_geodata['countrycode']},
-                        'testName': klass.name,
-                        'testVersion': klass.version,
+                        'testName': options['name'],
+                        'testVersion': options['version'],
                         }
         self.writeYamlLine(test_details)
         self._writeln('')
diff --git a/ooni/runner.py b/ooni/runner.py
index d711153..acf5519 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -18,20 +18,13 @@ from twisted.trial.runner import isTestCase
 from twisted.trial.runner import filenameToModule
 
 from ooni.inputunit import InputUnitFactory
-from ooni.nettest import InputTestSuite, NetTestCase
+from ooni.nettest import InputTestSuite
 from ooni.plugoo import tests as oonitests
 from ooni.reporter import ReporterFactory
 from ooni.utils import log, date
 from ooni.utils.legacy import LegacyOONITest
 from ooni.utils.legacy import start_legacy_test, adapt_legacy_test
 
-
-def isTemplate(obj):
-    origin = obj.__module__
-    if origin.find('templates') >= 0:
-        return True
-    return False
-
 def isLegacyTest(obj):
     """
     Returns True if the test in question is written using the OONITest legacy
@@ -43,6 +36,58 @@ def isLegacyTest(obj):
     except TypeError:
         return False
 
+def processTest(obj, config):
+    """
+    Process the parameters and :class:`twisted.python.usage.Options` of a
+    :class:`ooni.nettest.Nettest`.
+
+    :param obj:
+        An uninstantiated old test, which should be a subclass of
+        :class:`ooni.plugoo.tests.OONITest`.
+    :param config:
+        A configured and instantiated :class:`twisted.python.usage.Options`
+        class.
+    """
+
+    input_file = obj.inputFile
+    if obj.requiresRoot:
+        if os.getuid() != 0:
+            raise Exception("This test requires root to run")
+
+    if obj.optParameters or input_file \
+            or obj.advancedOptParameters or obj.optFlags:
+
+        if not obj.optParameters:
+            obj.optParameters = []
+
+        if input_file:
+            obj.optParameters.append(input_file)
+
+        if obj.advancedOptParameters:
+            log.debug("Got advanced options")
+            options = obj.advancedOptParameters()
+        else:
+            log.debug("Got optParameters")
+            class Options(usage.Options):
+                optParameters = obj.optParameters
+                if obj.optFlags:
+                    log.debug("Got optFlags")
+                    optFlags = obj.optFlags
+            options = Options()
+
+        options.parseOptions(config['subArgs'])
+        obj.localOptions = options
+
+        if input_file:
+            obj.inputFile = options[input_file[0]]
+        try:
+            tmp_obj = obj()
+            tmp_obj.getOptions()
+        except usage.UsageError:
+            options.opt_help()
+
+    return obj
+
 def findTestClassesFromConfig(config):
     """
     Takes as input the command line config parameters and returns the test
@@ -57,174 +102,76 @@ def findTestClassesFromConfig(config):
         A list of class objects found in a file or module given on the
         commandline.
     """
+
     filename = config['test']
     classes = []
 
     module = filenameToModule(filename)
     for name, val in inspect.getmembers(module):
         if isTestCase(val):
-            if val != NetTestCase and not isTemplate(val):
-                log.debug("findTestClassesFromConfig: detected %s"
-                          % val.__name__)
-                classes.append(val)
+            log.debug("Detected TestCase %s" % val)
+            classes.append(processTest(val, config))
         elif isLegacyTest(val):
             log.debug("Detected Legacy Test %s" % val)
             classes.append(adapt_legacy_test(val, config))
     return classes
 
-def makeTestCases(klass, tests, method_prefix=None):
+def makeTestCases(klass, tests, method_prefix):
     """
     Takes a class some tests and returns the test cases. method_prefix is how
     the test case functions should be prefixed with.
     """
-    if not method_prefix:
-        method_prefix = 'test'
 
     cases = []
     for test in tests:
-        log.debug("makeTestCases: making test case for %s" % test)
-        method_name = str(method_prefix)+str(test)
-        log.debug("makeTestCases: using methodName=%s" % method_name)
-        cases.append(klass(methodName=method_name))
+        cases.append(klass(method_prefix+test))
     return cases
 
-def getTestOptions(cls, subargs):
-    """
-    Process the parameters and :class:`twisted.python.usage.Options` of a
-    :class:`ooni.nettest.Nettest`.
-
-    :param cls:
-        An subclass of :class:`ooni.nettest.NetTestCase`.
-    :param config:
-        A configured and instantiated :class:`twisted.python.usage.Options`
-        class.
-    """
-    if cls.requiresRoot:
-        if os.getuid() != 0:
-            raise Exception("This test requires root to run")
-
-    try:
-        cls.buildOptions(subargs)
-    except Exception, e:
-        log.err(e)
-
-    return cls.local_options
-
 def loadTestsAndOptions(classes, config):
     """
     Takes a list of test classes and returns their testcases and options.
     Legacy tests will be adapted.
     """
-    from inspect import isclass
 
     method_prefix = 'test'
     options = []
     test_cases = []
 
-    DEPRECATED = LegacyOONITest
+    _old_klass_type = LegacyOONITest
 
     for klass in classes:
-        if isinstance(klass, DEPRECATED):
+        if isinstance(klass, _old_klass_type):
             try:
-                cases, opts = processLegacyTest(klass, config)
+                cases = start_legacy_test(klass)
                 if cases:
-                    log.debug("loadTestsAndOptions: processing cases %s"
-                              % str(cases))
+                    log.debug("Processing cases")
+                    log.debug(str(cases))
                     return [], []
                 test_cases.append(cases)
-            except Exception, e: log.err(e)
+            except Exception, e:
+                log.err(e)
             else:
                 try:
                     opts = klass.local_options
-                    option.append(opts)
+                    options.append(opts)
                 except AttributeError, ae:
                     options.append([])
                     log.err(ae)
-
-        elif issubclass(klass, NetTestCase):
-            try:
-                cases, opts = processNetTest(klass, config, method_prefix)
-            except Exception, e:
-                log.err(e)
-            else:
+        else:
+            tests = reflect.prefixedMethodNames(klass, method_prefix)
+            if tests:
+                cases = makeTestCases(klass, tests, method_prefix)
                 test_cases.append(cases)
+            try:
+                k = klass()
+                opts = k.getOptions()
                 options.append(opts)
+            except AttributeError, ae:
+                options.append([])
+                log.err(ae)
 
     return test_cases, options
 
-def processNetTest(klass, config, method_prefix):
-    try:
-        klass.setUpClass()
-    except Exception, e:
-        log.err(e)
-
-    subargs_from_config = config['subArgs']
-    log.debug("processNetTest: received subargs from config: %s"
-              % str(subargs_from_config))
-    try:
-        opts = getTestOptions(klass, subargs_from_config)
-    except Exception, e:
-        opts = []
-        log.err(e)
-
-    try:
-        log.debug("processNetTest: processing cases for %s"
-                  % (klass.name if hasattr(klass, 'name') else 'Network Test'))
-        tests = reflect.prefixedMethodNames(klass, method_prefix)
-    except Exception, e:
-        cases = []
-        opts = []
-        log.err(e)
-    else:
-        if tests:
-            cases = makeTestCases(klass, tests, method_prefix)
-            log.debug("processNetTest: test %s found cases %s"
-                      % (tests, cases))
-        else:
-            cases = []
-
-    return cases, opts
-
-def processLegacyTest(klass, config):
-    log.msg("Processing cases and options for legacy test %s"
-            % ( klass.shortName if hasattr(klass, shortName) else 'oonitest' ))
-    if hasattr(klass, description):
-        log.msg("%s" % klass.description)
-
-    subcmds = []
-    if hasattr(klass, options):        ## an unitiated Legacy test
-        log.debug("%s.options found: %s " % (klass, klass.options))
-        try:
-            assert isclass(klass.options), "legacytest.options is not class"
-        except AssertionError, ae:
-            log.debug(ae)
-        else:
-            ok = klass.options
-            ok.parseArgs = lambda x: subcmds.append(x)
-            try:
-                opts = ok()
-                opts.parseOptions(config['subArgs'])
-            except Exception, e:
-                log.err(e)
-                opts = {}
-
-    elif hasattr(klass, local_options): ## we've been initialized already
-        log.debug("processLegacyTest: %s.local_options found" % str(klass))
-        try:
-            opts = klass.local_options
-        except AttributeError, ae: opts = {}; log.err(ae)
-        log.debug("processLegacyTest: opts set to %s" % str(opts))
-
-    try:
-        cases = start_legacy_test(klass)
-        ## XXX we need to get these results into the reporter
-        if cases:
-            log.debug("processLegacyTest: found cases: %s" % str(cases))
-            return [], []
-    except Exception, e: cases = []; log.err(e)
-
-    return cases, opts
-
 class ORunner(object):
     """
     This is a specialized runner used by the ooniprobe command line tool.
@@ -237,29 +184,31 @@ class ORunner(object):
         self.cases = cases
         self.options = options
 
-        log.debug("ORunner: cases=%s" % type(cases))
-        log.debug("ORunner: options=%s" % options)
-
-
         try:
-            first = options.pop(0)
-        except:
-            first = options
-
-        if 'inputs' in first:
-            self.inputs = self.options['inputs']
+            assert len(options) != 0, "Length of options is zero!"
+        except AssertionError, ae:
+            self.inputs = []
+            log.err(ae)
         else:
-            log.msg("Could not find inputs!")
-            self.inputs = [None]
+            try:
+                first = options.pop(0)
+            except:
+                first = options
+
+            if 'inputs' in first:
+                self.inputs = options['inputs']
+            else:
+                log.msg("Could not find inputs!")
+                log.msg("options[0] = %s" % first)
+                self.inputs = [None]
 
         try:
             reportFile = open(config['reportfile'], 'a+')
         except:
             filename = 'report_'+date.timestamp()+'.yaml'
             reportFile = open(filename, 'a+')
-        self.reporterFactory = ReporterFactory(
-            reportFile, testSuite=self.baseSuite(self.cases)
-            )
+        self.reporterFactory = ReporterFactory(reportFile,
+                                               testSuite=self.baseSuite(self.cases))
 
     def runWithInputUnit(self, input_unit):
         idx = 0
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index acad538..2e2d9e1 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -6,12 +6,14 @@
 import random
 
 from zope.interface import implements
+
 from twisted.python import usage
 from twisted.plugin import IPlugin
 from twisted.internet import protocol, defer
 from twisted.internet.ssl import ClientContextFactory
 
 from twisted.web.http_headers import Headers
+from twisted.web.iweb import IBodyProducer
 
 from ooni.nettest import NetTestCase
 from ooni.utils import log
@@ -28,6 +30,23 @@ useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Geck
               ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"),
               ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")]
 
+class StringProducer(object):
+    implements(IBodyProducer)
+
+    def __init__(self, body):
+        self.body = body
+        self.length = len(body)
+
+    def startProducing(self, consumer):
+        consumer.write(self.body)
+        return defer.succeed(None)
+
+    def pauseProducing(self):
+        pass
+
+    def stopProducing(self):
+        pass
+
 class BodyReceiver(protocol.Protocol):
     def __init__(self, finished):
         self.finished = finished
diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py
index 9c80f62..fe97ca3 100644
--- a/ooni/utils/onion.py
+++ b/ooni/utils/onion.py
@@ -21,8 +21,8 @@ import sys
 from twisted.internet  import defer
 from zope.interface    import implements
 
-from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher
-from ooni.lib.txtorcon import TorState, TorConfig
+from txtorcon          import CircuitListenerMixin, IStreamAttacher
+from txtorcon          import TorState, TorConfig
 from ooni.utils        import log
 from ooni.utils.timer  import deferred_timeout, TimeoutError
 

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits