[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Implement a working ooniclient based off of trial



commit 603d5cdc27a13aef4d9d25ffbd9108fb3bdbdcac
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Tue Sep 11 15:35:21 2012 +0000

    Implement a working ooniclient based off of trial
---
 ooni/nettest.py      |   97 +++++++++++++++++++++++++++
 ooni/oonicli.py      |   73 ++++-----------------
 ooni/plugoo/tests.py |    1 +
 ooni/reporter.py     |   16 ++---
 ooni/runner.py       |  177 +++++++++++++++++++++++++++++++++++++++++++++++++-
 5 files changed, 292 insertions(+), 72 deletions(-)

diff --git a/ooni/nettest.py b/ooni/nettest.py
new file mode 100644
index 0000000..fe8c05c
--- /dev/null
+++ b/ooni/nettest.py
@@ -0,0 +1,97 @@
+
+from twisted.python import log
+from twisted.trial import unittest, itrial
+
+pyunit = __import__('unittest')
+
+def _iterateTests(testSuiteOrCase):
+    """
+    Iterate through all of the test cases in C{testSuiteOrCase}.
+    """
+    try:
+        suite = iter(testSuiteOrCase)
+    except TypeError:
+        if not testSuiteOrCase.inputs:
+            yield testSuiteOrCase
+        else:
+            inputs = iter(testSuiteOrCase.inputs)
+            print "Detected Sub shit! %s" % inputs
+            for input in inputs:
+                yield testSuiteOrCase, input
+    else:
+        for test in suite:
+            for subtest in _iterateTests(test):
+                yield subtest
+
+
+class TestCase(unittest.TestCase):
+    """
+    A test case represents the minimum
+    """
+    def run(self, result, input):
+        """
+        Run the test case, storing the results in C{result}.
+
+        First runs C{setUp} on self, then runs the test method (defined in the
+        constructor), then runs C{tearDown}.  As with the standard library
+        L{unittest.TestCase}, the return value of these methods is disregarded.
+        In particular, returning a L{Deferred} has no special additional
+        consequences.
+
+        @param result: A L{TestResult} object.
+        """
+        log.msg("--> %s <--" % (self.id()))
+        new_result = itrial.IReporter(result, None)
+        if new_result is None:
+            result = PyUnitResultAdapter(result)
+        else:
+            result = new_result
+        result.startTest(self)
+        if self.getSkip(): # don't run test methods that are marked as .skip
+            result.addSkip(self, self.getSkip())
+            result.stopTest(self)
+            return
+
+        self._passed = False
+        self._warnings = []
+
+        self._installObserver()
+        # All the code inside _runFixturesAndTest will be run such that warnings
+        # emitted by it will be collected and retrievable by flushWarnings.
+        unittest._collectWarnings(self._warnings.append, self._runFixturesAndTest, result)
+
+        # Any collected warnings which the test method didn't flush get
+        # re-emitted so they'll be logged or show up on stdout or whatever.
+        for w in self.flushWarnings():
+            try:
+                warnings.warn_explicit(**w)
+            except:
+                result.addError(self, failure.Failure())
+
+        result.stopTest(self)
+
+
+class TestSuite(pyunit.TestSuite):
+    """
+    Extend the standard library's C{TestSuite} with support for the visitor
+    pattern and a consistently overrideable C{run} method.
+    """
+
+    def __call__(self, result, input):
+        return self.run(result, input)
+
+
+    def run(self, result, input):
+        """
+        Call C{run} on every member of the suite.
+        """
+        # we implement this because Python 2.3 unittest defines this code
+        # in __call__, whereas 2.4 defines the code in run.
+        for test in self._tests:
+            if result.shouldStop:
+                break
+            print test
+            print "----------------"
+            test(result, input)
+        return result
+
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 8ace160..199b4d4 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -22,19 +22,8 @@ from twisted.python.filepath import FilePath
 from twisted import plugin
 from twisted.python.util import spewer
 from twisted.python.compat import set
-from twisted.trial import runner, itrial, reporter
-
-
-# Yea, this is stupid.  Leave it for for command-line compatibility for a
-# while, though.
-TBFORMAT_MAP = {
-    'plain': 'default',
-    'default': 'default',
-    'emacs': 'brief',
-    'brief': 'brief',
-    'cgitb': 'verbose',
-    'verbose': 'verbose'
-    }
+from twisted.trial import itrial
+from ooni import runner, reporter
 
 
 def _parseLocalVariables(line):
@@ -98,26 +87,6 @@ def isTestFile(filename):
             and os.path.splitext(basename)[1] == ('.py'))
 
 
-def _reporterAction():
-    return usage.CompleteList([p.longOpt for p in
-                               plugin.getPlugins(itrial.IReporter)])
-
-class Options(usage.Options):
-
-    optParameters = [
-        ['parallelism', 'n', 10, "Specify the number of parallel tests to run"],
-        ['output', 'o', 'report.log', "Specify output report file"],
-        ['log', 'l', 'oonicli.log', "Specify output log file"]
-    ]
-
-    def opt_version(self):
-        """
-        Display OONI version and exit.
-        """
-        print "OONI version:", __version__
-        sys.exit(0)
-
-
 class Options(usage.Options, app.ReactorSelectionMixin):
     synopsis = """%s [options] [[file|package|module|TestCase|testmethod]...]
     """ % (os.path.basename(sys.argv[0]),)
@@ -136,12 +105,11 @@ class Options(usage.Options, app.ReactorSelectionMixin):
                 ["nopm", None, "don't automatically jump into debugger for "
                  "postmorteming of exceptions"],
                 ["dry-run", 'n', "do everything but run the tests"],
-                ["force-gc", None, "Have Trial run gc.collect() before and "
+                ["force-gc", None, "Have OONI run gc.collect() before and "
                  "after each test case."],
                 ["profile", None, "Run tests under the Python profiler"],
                 ["unclean-warnings", None,
                  "Turn dirty reactor errors into warnings"],
-                ["until-failure", "u", "Repeat test until it fails"],
                 ["no-recurse", "N", "Don't recurse into packages"],
                 ['help-reporters', None,
                  "Help on available output plugins (reporters)"]
@@ -153,13 +121,12 @@ class Options(usage.Options, app.ReactorSelectionMixin):
          "Run tests in random order using the specified seed"],
         ['temp-directory', None, '_trial_temp',
          'Path to use as working directory for tests.'],
-        ['reporter', None, 'verbose',
+        ['reporter', None, 'default',
          'The reporter to use for this test run.  See --help-reporters for '
          'more info.']]
 
     compData = usage.Completions(
         optActions={"tbformat": usage.CompleteList(["plain", "emacs", "cgitb"]),
-                    "reporter": _reporterAction,
                     "logfile": usage.CompleteFiles(descr="log file name"),
                     "random": usage.Completer(descr="random seed")},
         extraActions=[usage.CompleteFiles(
@@ -167,7 +134,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):
                 repeat=True)],
         )
 
-    fallbackReporter = reporter.TreeReporter
+    fallbackReporter = reporter.OONIReporter
     tracer = None
 
     def __init__(self):
@@ -208,7 +175,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):
         # value to the test suite as a module.
         #
         # This parameter allows automated processes (like Buildbot) to pass
-        # a list of files to Trial with the general expectation of "these files,
+        # a list of files to OONI with the general expectation of "these files,
         # whatever they are, will get tested"
         if not os.path.isfile(filename):
             sys.stderr.write("File %r doesn't exist\n" % (filename,))
@@ -300,21 +267,11 @@ class Options(usage.Options, app.ReactorSelectionMixin):
         self['tests'].update(args)
 
 
-    def _loadReporterByName(self, name):
-        for p in plugin.getPlugins(itrial.IReporter):
-            qual = "%s.%s" % (p.module, p.klass)
-            if p.longOpt == name:
-                return reflect.namedAny(qual)
-        raise usage.UsageError("Only pass names of Reporter plugins to "
-                               "--reporter. See --help-reporters for "
-                               "more info.")
-
-
     def postOptions(self):
         # Only load reporters now, as opposed to any earlier, to avoid letting
         # application-defined plugins muck up reactor selecting by importing
         # t.i.reactor and causing the default to be installed.
-        self['reporter'] = self._loadReporterByName(self['reporter'])
+        self['reporter'] = reporter.OONIReporter
 
         if 'tbformat' not in self:
             self['tbformat'] = 'default'
@@ -338,10 +295,10 @@ def _initialDebugSetup(config):
 def _getSuite(config):
     loader = _getLoader(config)
     recurse = not config['no-recurse']
+    print "loadByNames %s" % config['tests']
     return loader.loadByNames(config['tests'], recurse)
 
 
-
 def _getLoader(config):
     loader = runner.TestLoader()
     if config['random']:
@@ -349,8 +306,6 @@ def _getLoader(config):
         randomer.seed(config['random'])
         loader.sorter = lambda x : randomer.random()
         print 'Running tests shuffled with seed %d\n' % config['random']
-    if not config['until-failure']:
-        loader.suiteFactory = runner.DestructiveTestSuite
     return loader
 
 
@@ -358,10 +313,11 @@ def _getLoader(config):
 def _makeRunner(config):
     mode = None
     if config['debug']:
-        mode = runner.TrialRunner.DEBUG
+        mode = runner.OONIRunner.DEBUG
     if config['dry-run']:
-        mode = runner.TrialRunner.DRY_RUN
-    return runner.TrialRunner(config['reporter'],
+        mode = runner.OONIRunner.DRY_RUN
+    print "using %s" % config['reporter']
+    return runner.OONIRunner(config['reporter'],
                               mode=mode,
                               profile=config['profile'],
                               logfile=config['logfile'],
@@ -384,10 +340,7 @@ def run():
     _initialDebugSetup(config)
     trialRunner = _makeRunner(config)
     suite = _getSuite(config)
-    if config['until-failure']:
-        test_result = trialRunner.runUntilFailure(suite)
-    else:
-        test_result = trialRunner.run(suite)
+    test_result = trialRunner.run(suite)
     if config.tracer:
         sys.settrace(None)
         results = config.tracer.results()
diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py
index 5fad85e..2b1e87c 100644
--- a/ooni/plugoo/tests.py
+++ b/ooni/plugoo/tests.py
@@ -22,6 +22,7 @@ class OONITest(object):
     developer to benefit from OONIs reporting system and command line argument
     parsing system.
     """
+    name = "oonitest"
     # By default we set this to False, meaning that we don't block
     blocking = False
     reactor = None
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 0ecf2ea..d20160f 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -1,13 +1,11 @@
 from twisted.trial import reporter
 
-class TestResult(reporter.TestResult):
-    """
-    Accumulates the results of several ooni.nettest.TestCases.
-
-    The output format of a TestResult is YAML and it will contain all the basic
-    information that a test result should contain.
-    """
-    def __init__(self):
-        super(TestResult, self).__init__()
+class OONIReporter(reporter.Reporter):
+
+    def startTest(self, test, input=None):
+        print "Running %s" % test
+        print "Input %s" % input
+        self._input = input
+        super(OONIReporter, self).startTest(test)
 
 
diff --git a/ooni/runner.py b/ooni/runner.py
index d7caa9d..c6ad90b 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -1,7 +1,178 @@
-from twisted.trial import runner
+import types
+import time
+import inspect
 
+from twisted.internet import defer
+from twisted.trial import unittest
+from twisted.trial.runner import TrialRunner, TestLoader
+from twisted.trial.runner import isPackage, isTestCase
 
-class TestLoader(runner.TestLoader):
-    pass
+from ooni import nettest
+from ooni.plugoo import tests as oonitests
+
+def isLegacyTest(obj):
+    """
+    Returns True if the test in question is written using the OONITest legacy
+    class.
+    We do this for backward compatibility of the OONIProbe API.
+    """
+    try:
+        return issubclass(obj, oonitests.OONITest)
+    except TypeError:
+        return False
+
+def adaptLegacyTest(obj):
+    """
+    We take a legacy OONITest class and convert it into a nettest.TestCase.
+    This allows backward compatibility of old OONI tests.
+
+    XXX perhaps we could implement another extra layer that makes the even
+    older test cases compatible with the new OONI.
+    """
+    class LegacyOONITest(nettest.TestCase):
+        pass
+
+
+class LoggedSuite(nettest.TestSuite):
+    """
+    Any errors logged in this suite will be reported to the L{TestResult}
+    object.
+    """
+
+    def run(self, result, input):
+        """
+        Run the suite, storing all errors in C{result}. If an error is logged
+        while no tests are running, then it will be added as an error to
+        C{result}.
+
+        @param result: A L{TestResult} object.
+        """
+        observer = unittest._logObserver
+        observer._add()
+        super(LoggedSuite, self).run(result, input)
+        observer._remove()
+        for error in observer.getErrors():
+            result.addError(TestHolder(NOT_IN_TEST), error)
+        observer.flushErrors()
+
+
+class OONISuite(nettest.TestSuite):
+    """
+    Suite to wrap around every single test in a C{trial} run. Used internally
+    by OONI to set up things necessary for OONI tests to work, regardless of
+    what context they are run in.
+    """
+
+    def __init__(self, tests=()):
+        suite = LoggedSuite(tests)
+        super(OONISuite, self).__init__([suite])
+
+    def _bail(self):
+        from twisted.internet import reactor
+        d = defer.Deferred()
+        reactor.addSystemEventTrigger('after', 'shutdown',
+                                      lambda: d.callback(None))
+        reactor.fireSystemEvent('shutdown') # radix's suggestion
+        # As long as TestCase does crap stuff with the reactor we need to
+        # manually shutdown the reactor here, and that requires util.wait
+        # :(
+        # so that the shutdown event completes
+        nettest.TestCase('mktemp')._wait(d)
+
+    def run(self, result, input):
+        try:
+            nettest.TestSuite.run(self, result, input)
+        finally:
+            self._bail()
+
+
+class OONIRunner(TrialRunner):
+    def run(self, test):
+        return TrialRunner.run(self, test)
+
+    def _runWithoutDecoration(self, test):
+        """
+        Private helper that runs the given test but doesn't decorate it.
+        """
+        result = self._makeResult()
+        # decorate the suite with reactor cleanup and log starting
+        # This should move out of the runner and be presumed to be
+        # present
+        suite = OONISuite([test])
+        print "HERE IS THE TEST:"
+        print test
+        print "-------------"
+        try:
+            inputs = test.inputs
+        except:
+            inputs = [None]
+
+        startTime = time.time()
+        if self.mode == self.DRY_RUN:
+            for single in nettest._iterateTests(suite):
+                input = None
+                if type(single) == type(tuple()):
+                    single, input = single
+                result.startTest(single, input)
+                result.addSuccess(single)
+                result.stopTest(single)
+        else:
+            if self.mode == self.DEBUG:
+                # open question - should this be self.debug() instead.
+                debugger = self._getDebugger()
+                run = lambda x: debugger.runcall(suite.run, result, x)
+            else:
+                run = lambda x: suite.run(result, x)
+
+            oldDir = self._setUpTestdir()
+            try:
+                self._setUpLogFile()
+                # XXX work on this better
+                for input in inputs:
+                    run(input)
+            finally:
+                self._tearDownLogFile()
+                self._tearDownTestdir(oldDir)
+
+        endTime = time.time()
+        done = getattr(result, 'done', None)
+        if done is None:
+            warnings.warn(
+                "%s should implement done() but doesn't. Falling back to "
+                "printErrors() and friends." % reflect.qual(result.__class__),
+                category=DeprecationWarning, stacklevel=3)
+            result.printErrors()
+            result.writeln(result.separator)
+            result.writeln('Ran %d tests in %.3fs', result.testsRun,
+                           endTime - startTime)
+            result.write('\n')
+            result.printSummary()
+        else:
+            result.done()
+        return result
+
+
+class TestLoader(TestLoader):
+    """
+    Reponsible for finding the modules that can work as tests and running them.
+    If we detect that a certain test is written using the legacy OONI API we
+    will wrap it around a next gen class to make it work here too.
+    """
+    def __init__(self):
+        super(TestLoader, self).__init__()
+        self.suiteFactory = nettest.TestSuite
+
+    def findTestClasses(self, module):
+        classes = []
+        for name, val in inspect.getmembers(module):
+            if isTestCase(val):
+                classes.append(val)
+            # This is here to allow backward compatibility with legacy OONI
+            # tests.
+            elif isLegacyTest(val):
+                #val = adaptLegacyTest(val)
+                classes.append(val)
+        return self.sort(classes)
+        #return runner.TestLoader.findTestClasses(self, module)
 
 



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits