[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Added timeout and abort for test inputs, methods, and classes.



commit 4e52ebe138393a3d942460890047b24ff467fdbc
Author: Isis Lovecruft <isis@xxxxxxxxxxxxxx>
Date:   Mon Dec 3 17:47:32 2012 +0000

    Added timeout and abort for test inputs, methods, and classes.
---
 nettests/bridge_reachability/tcpsyn.py |    2 +-
 ooni/nettest.py                        |  115 ++++++++++++++++++++++++++-----
 ooni/runner.py                         |  103 ++++++++++++++++++++++++----
 3 files changed, 186 insertions(+), 34 deletions(-)

diff --git a/nettests/bridge_reachability/tcpsyn.py b/nettests/bridge_reachability/tcpsyn.py
index 39c882b..f92ab09 100644
--- a/nettests/bridge_reachability/tcpsyn.py
+++ b/nettests/bridge_reachability/tcpsyn.py
@@ -63,7 +63,7 @@ class TCPFlagTest(nettest.NetTestCase):
 
     #destinations = {}
 
-    @log.catcher
+    @log.catch
     def setUp(self, *a, **kw):
         """Configure commandline parameters for TCPSynTest."""
         self.report = {}
diff --git a/ooni/nettest.py b/ooni/nettest.py
index a868ac4..139b4f4 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -13,12 +13,26 @@ import os
 import itertools
 import traceback
 
-from twisted.trial import unittest, itrial, util
+from twisted.trial import unittest, itrial
+from twisted.trial import util as txtrutil
 from twisted.internet import defer, utils
 from twisted.python import usage
 
+from ooni import runner
 from ooni.utils import log
 
+# This needs to be here so that NetTestCase.abort() can call it, since we
+# cannot import runner because runner imports NetTestCase.
+def isTestCase(obj):
+    """
+    Return True if obj is a subclass of NetTestCase, false if otherwise.
+    """
+    try:
+        return issubclass(obj, NetTestCase)
+    except TypeError:
+        return False
+
+
 class NetTestCase(object):
     """
     This is the base of the OONI nettest universe. When you write a nettest
@@ -61,21 +75,28 @@ class NetTestCase(object):
 
     * requiresRoot: set to True if the test must be run as root.
 
-    * optFlags: is assigned a list of lists. Each list represents a flag parameter, as so:
+    * optFlags: 
+          is assigned a list of lists. Each list represents a flag
+          parameter, as so:
 
-        optFlags = [['verbose', 'v', 'Makes it tell you what it doing.'], | ['quiet', 'q', 'Be vewy vewy quiet.']]
+        optFlags = [
+            ['verbose', 'v', 'Makes it tell you what it doing.'],
+            ['quiet', 'q', 'Be vewy vewy quiet.']]
 
     As you can see, the first item is the long option name (prefixed with
     '--' on the command line), followed by the short option name (prefixed with
     '-'), and the description. The description is used for the built-in handling of
     the --help switch, which prints a usage summary.
 
+    * optParameters: 
+          is much the same, except the list also contains a default value:
 
-    * optParameters: is much the same, except the list also contains a default value:
-
-        | optParameters = [['outfile', 'O', 'outfile.log', 'Description...']]
+        optParameters = [
+            ['outfile', 'O', 'outfile.log', 'Description...']]
 
-    * usageOptions: a subclass of twisted.python.usage.Options for more advanced command line arguments fun.
+    * usageOptions: 
+          a subclass of twisted.python.usage.Options for more advanced command
+          line arguments fun.
 
     * requiredOptions: a list containing the name of the options that are
                        required for proper running of a test.
@@ -97,22 +118,18 @@ class NetTestCase(object):
 
     optFlags = None
     optParameters = None
-
     usageOptions = None
     requiredOptions = []
     requiresRoot = False
 
     localOptions = {}
+
     def _setUp(self):
-        """
-        This is the internal setup method to be overwritten by templates.
-        """
+        """This is the internal setup method to be overwritten by templates."""
         pass
 
     def setUp(self):
-        """
-        Place here your logic to be executed when the test is being setup.
-        """
+        """Place your logic to be executed when the test is being setup here."""
         pass
 
     def inputProcessor(self, filename=None):
@@ -166,8 +183,68 @@ class NetTestCase(object):
     def __repr__(self):
         return "<%s inputs=%s>" % (self.__class__, self.inputs)
 
-    def __test_done__(self):
-        up = inspect.stack()
-        parent = up[1]
-        # XXX call oreporter.allDone() from parent stack frame
-        raise NotImplemented
+    def _getSkip(self):
+        return txtrutil.acquireAttribute(self._parents, 'skip', None)
+
+    #def _getSkipReason(self, method, skip):
+    #    return super(TestCase, self)._getSkipReason(self, method, skip)
+
+    def _getTimeout(self):
+        """
+        Returns the timeout value set on this test. Check on the instance
+        first, the the class, then the module, then package. As soon as it
+        finds something with a timeout attribute, returns that. Returns
+        twisted.trial.util.DEFAULT_TIMEOUT_DURATION if it cannot find
+        anything. See TestCase docstring for more details.
+        """
+        testMethod = getattr(self, methodName)
+        self._parents = [testMethod, self]
+        self._parents.extend(txtrutil.getPythonContainers(testMethod))
+        timeout = txtrutil.acquireAttribute(self._parents, 'timeout', 
+                                            txtrutil.DEFAULT_TIMEOUT_DURATION)
+        try:
+            return float(timeout)
+        except (ValueError, TypeError):
+            warnings.warn("'timeout' attribute needs to be a number.",
+                          category=DeprecationWarning)
+            return txtrutil.DEFAULT_TIMEOUT_DURATION
+
+    def _abort(self, reason, obj=None):
+        """
+        Abort running an input, test_method, or test_class. If called with only
+        one argument, assume we're going to ignore the current input. Otherwise,
+        the name of the method or class in relation to the test_instance,
+        i.e. "self" should be given as value for the keyword argument "obj".
+
+        XXX call oreporter.allDone() from parent stack frame
+        """
+        reason = str(reason) # XXX should probably coerce
+        raise SkipTest("%s\n%s" % (str(reason), str(self.input)) )
+
+    def _abortMethod(self, reason, method):
+        if inspect.ismethod(method):
+            abort = getattr(self.__class__, method, False)
+            log.debug("Aborting remaining inputs for %s" % str(abort.func_name))
+            setattr(abort, 'skip', reason)
+        else:
+            log.debug("abortMethod(): could not find method %s" % str(method))
+
+    @log.catch
+    def _abortClass(self, reason, cls):
+        if not inspect.isclass(obj) or not runner.isTestCase(obj):
+            log.debug("_abortClass() could not find class %s" % str(cls))
+            return
+        abort = getattr(obj, '__class__', self.__class__)
+        log.debug("Aborting %s test" % str(abort.name))
+        setattr(abort, 'skip', reason)
+
+    def abortCurrentInput(self, reason):
+        """
+        Abort the current input.
+        
+        @param reason: A string explaining why this test is being skipped.
+        """
+        return self._abort(reason)
+
+    def abortInput(self, reason):
+        return self._abort(reason)
diff --git a/ooni/runner.py b/ooni/runner.py
index 8d750ae..b6de21e 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -15,13 +15,14 @@ import inspect
 import traceback
 import itertools
 
-from twisted.python import reflect, usage
+from twisted.python import reflect, usage, failure
 from twisted.internet import defer
 from twisted.trial.runner import filenameToModule
+from twisted.trial import util as txtrutil
 from twisted.internet import reactor, threads
 
 from ooni.inputunit import InputUnitFactory
-from ooni.nettest import NetTestCase
+from ooni.nettest import NetTestCase, isTestCase
 
 from ooni import reporter
 
@@ -92,12 +93,6 @@ def processTest(obj, cmd_line_options):
 
     return obj
 
-def isTestCase(obj):
-    try:
-        return issubclass(obj, NetTestCase)
-    except TypeError:
-        return False
-
 def findTestClassesFromConfig(cmd_line_options):
     """
     Takes as input the command line config parameters and returns the test
@@ -112,7 +107,6 @@ def findTestClassesFromConfig(cmd_line_options):
         A list of class objects found in a file or module given on the
         commandline.
     """
-
     filename = cmd_line_options['test']
     classes = []
 
@@ -150,12 +144,64 @@ def loadTestsAndOptions(classes, cmd_line_options):
 
     return test_cases, options
 
+def abortTestRun(test_class, warn_err_fail, test_input, oreporter):
+    """
+    Abort the entire test, and record the error, failure, or warning for why
+    it could not be completed.
+    """
+    log.warn("Aborting remaining tests for %s" % test_name)
+
+def abortTestWasCalled(abort_reason, abort_what, test_class, test_instance, 
+                       test_method, test_input, oreporter):
+    """
+    XXX
+    """
+    if not abort_what in ['class', 'method', 'input']:
+        log.warn("__test_abort__() must specify 'class', 'method', or 'input'")
+        abort_what = 'input'    
+
+    if not isinstance(abort_reason, Exception):
+        abort_reason = Exception(str(abort_reason))
+    if abort_what == 'input':
+        log.msg("%s test requested to abort for input: %s"
+                % (test_instance.name, test_input))
+        d = maybeDeferred()
+
+    if hasattr(test_instance, "abort_all"):
+        log.msg("%s test requested to abort all remaining inputs"
+                % test_instance.name)
+    else:
+        d = defer.Deferred()
+        d.cancel()
+        d = abortTestRun(test_class, reason, test_input, oreporter)
+    
+
 def runTestWithInput(test_class, test_method, test_input, oreporter):
     """
     Runs a single testcase from a NetTestCase with one input.
     """
     log.debug("Running %s with %s" % (test_method, test_input))
 
+    def test_abort_single_input(reason, test_instance, test_name):
+        pass
+
+    def test_timeout(d):
+        err = defer.TimeoutError("%s test for %s timed out after %s seconds"
+                                 % (test_name, test_instance.input, 
+                                    test_instance.timeout))
+        fail = failure.Failure(err)
+        try:
+            d.errback(fail)
+        except defer.AlreadyCalledError:
+            # if the deferred has already been called but the *back chain is
+            # still unfinished, crash the reactor and report the timeout
+            reactor.crash()
+            test_instance._timedOut = True    # see test_instance._wait
+            # XXX result is TestResult utils? 
+            RESULT.addExpectedFailure(test_instance, fail)
+    test_timeout = utils.suppressWarnings(
+        test_timeout, util.suppress(category=DeprecationWarning))
+
     def test_done(result, test_instance, test_name):
         log.debug("runTestWithInput: concluded %s" % test_name)
         return oreporter.testDone(test_instance, test_name)
@@ -169,16 +215,45 @@ def runTestWithInput(test_class, test_method, test_input, oreporter):
     log.debug("Processing %s" % test_instance.name)
     # use this to keep track of the test runtime
     test_instance._start_time = time.time()
+    test_instance.timeout = test_instance._getTimeout()
     # call setups on the test
     test_instance._setUp()
     test_instance.setUp()
+    test_ignored = util.acquireAttribute(test_instance._parents, 'skip', None)
+
     test = getattr(test_instance, test_method)
 
-    d = defer.maybeDeferred(test)
-    d.addCallback(test_done, test_instance, test_method)
-    d.addErrback(test_error, test_instance, test_method)
-    log.debug("returning %s input" % test_method)
-    return d
+    # check if we've aborted
+    test_skip = test.getSkip()
+    if test_skip is not None:
+        log.debug("%s.getSkip() returned %s" % (str(test_class), 
+                                                str(test_skip)) )
+ 
+    abort_reason, abort_what = getattr(test_instance, 'abort', ('input', None))
+    if abort_reason is not None:
+        do_abort = abortTestWasCalled(abort_reason, abort_what, test_class,
+                                      test_instance, test_method, test_input,
+                                      oreporter)
+        return defer.maybeDeferred(do_abort)
+    else:
+        d = defer.maybeDeferred(test)
+
+        # register the timer with the reactor
+        call = reactor.callLater(test_timeout, test_timed_out, d)
+        d.addBoth(lambda x: call.active() and call.cancel() or x)
+
+        # XXX check if test called test_abort...
+        d.addCallbacks(test_abort, 
+                       test_error, 
+                       callbackArgs=(test_instance, test_method), 
+                       errbackArgs=(test_instance, test_method) )
+        d.addCallback(test_done, test_instance, test_method)
+        d.addErrback(test_error, test_instance, test_method)
+        log.debug("returning %s input" % test_method)
+
+        ignored = d.getSkip()
+
+        return d
 
 def runTestWithInputUnit(test_class, test_method, input_unit, oreporter):
     """



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits