[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Made some progress, not there yet.



commit 7dce07bfc58566d2fdec4eec8a831d4ecd545d5f
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Tue Sep 18 17:51:07 2012 +0000

    Made some progress, not there yet.
    * After this commit I am going to throw away a lot of code and start over
---
 ooni/input.py    |   10 ++-
 ooni/nettest.py  |   50 +++++++--
 ooni/oonicli.py  |   56 +++--------
 ooni/reporter.py |    5 +-
 ooni/runner.py   |  305 +++++++++--------------------------------------------
 5 files changed, 117 insertions(+), 309 deletions(-)

diff --git a/ooni/input.py b/ooni/input.py
index f534393..b931b82 100644
--- a/ooni/input.py
+++ b/ooni/input.py
@@ -43,7 +43,7 @@ class InputUnit(object):
     passed onto a TestCase.
     """
     def __init__(self, inputs=[]):
-        self._inputs = inputs
+        self._inputs = iter(inputs)
 
     def __repr__(self):
         return "<%s inputs=%s>" % (self.__class__, self._inputs)
@@ -53,7 +53,13 @@ class InputUnit(object):
             self._inputs.append(input)
 
     def __iter__(self):
-        return iter(self._inputs)
+        return self
+
+    def next(self):
+        try:
+            return self._inputs.next()
+        except:
+            raise StopIteration
 
     def append(self, input):
         self._inputs.append(input)
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 0c8858b..4ab1e0f 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -19,28 +19,60 @@ def _iterateTests(testSuiteOrCase):
                 yield subtest
 
 
+class TestSuiteFactory(object):
+    def __init__(self, inputUnit, tests, basesuite):
+        self._baseSuite = basesuite
+        self._inputUnit = inputUnit
+        self._idx = 0
+        self.tests = tests
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        try:
+            next_input = self._inputUnit.next()
+            print "Now dealing with %s %s" % (next_input, self._idx)
+        except:
+            raise StopIteration
+        new_test_suite = self._baseSuite(self.tests)
+        new_test_suite.input = next_input
+        new_test_suite._idx = self._idx
+
+        self._idx += 1
+        return new_test_suite
+
 class TestSuite(pyunit.TestSuite):
-    inputUnit = [None]
+    def __init__(self, tests=()):
+        self._tests = []
+        self.input = None
+        self._idx = 0
+        self.addTests(tests)
+
     def __repr__(self):
-        return "<%s input=%s, tests=%s>" % (self.__class__, self.inputUnit, list(self))
+        return "<%s input=%s tests=%s>" % (self.__class__,
+                self.input, self._tests)
 
-    def run(self, result, inputUnit=[None]):
+    def run(self, result):
         """
         Call C{run} on every member of the suite.
         """
         # we implement this because Python 2.3 unittest defines this code
         # in __call__, whereas 2.4 defines the code in run.
-        idx = 0
-        for input, test in itertools.product(inputUnit, self._tests):
+        for i, test in enumerate(self._tests):
             if result.shouldStop:
                 break
-            self.inputUnit = inputUnit
-            test.input = input
-            test.idx = idx
+            test.input = self.input
+            test._idx = self._idx + i
             test(result)
-            idx += 1
 
         return result
 
 class TestCase(unittest.TestCase):
     name = "DefaultTestName"
+    inputs = [None]
+
+    def __repr__(self):
+        return "<%s inputs=%s>" % (self.__class__, self.inputs)
+
+
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index fa7742f..be73d30 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -17,12 +17,13 @@ import sys, os, random, gc, time, warnings
 
 from twisted.internet import defer
 from twisted.application import app
-from twisted.python import usage, reflect, failure
+from twisted.python import usage, reflect, failure, log
 from twisted.python.filepath import FilePath
 from twisted import plugin
 from twisted.python.util import spewer
 from twisted.python.compat import set
 from twisted.trial import itrial
+from twisted.trial import runner as irunner
 from ooni import runner, reporter
 
 
@@ -104,10 +105,8 @@ class Options(usage.Options, app.ReactorSelectionMixin):
                  "callback stack traces"],
                 ["nopm", None, "don't automatically jump into debugger for "
                  "postmorteming of exceptions"],
-                ["dry-run", 'n', "do everything but run the tests"],
                 ["force-gc", None, "Have OONI run gc.collect() before and "
                  "after each test case."],
-                ["profile", None, "Run tests under the Python profiler"],
                 ["unclean-warnings", None,
                  "Turn dirty reactor errors into warnings"],
                 ["no-recurse", "N", "Don't recurse into packages"],
@@ -118,9 +117,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):
     optParameters = [
         ["reportfile", "o", "report.yaml", "report file name"],
         ["logfile", "l", "test.log", "log file name"],
-        ["random", "z", None,
-         "Run tests in random order using the specified seed"],
-        ['temp-directory', None, '_trial_temp',
+        ['temp-directory', None, '_ooni_temp',
          'Path to use as working directory for tests.'],
         ['reporter', None, 'default',
          'The reporter to use for this test run.  See --help-reporters for '
@@ -129,7 +126,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):
     compData = usage.Completions(
         optActions={"tbformat": usage.CompleteList(["plain", "emacs", "cgitb"]),
                     "logfile": usage.CompleteFiles(descr="log file name"),
-                    "random": usage.Completer(descr="random seed")},
+                    },
         extraActions=[usage.CompleteFiles(
                 "*.py", descr="file | module | package | TestCase | testMethod",
                 repeat=True)],
@@ -238,20 +235,6 @@ class Options(usage.Options, app.ReactorSelectionMixin):
                 "argument to recursionlimit must be an integer")
 
 
-    def opt_random(self, option):
-        try:
-            self['random'] = long(option)
-        except ValueError:
-            raise usage.UsageError(
-                "Argument to --random must be a positive integer")
-        else:
-            if self['random'] < 0:
-                raise usage.UsageError(
-                    "Argument to --random must be a positive integer")
-            elif self['random'] == 0:
-                self['random'] = long(time.time() * 100)
-
-
     def opt_without_module(self, option):
         """
         Fake the lack of the specified modules, separated with commas.
@@ -283,7 +266,6 @@ class Options(usage.Options, app.ReactorSelectionMixin):
             failure.DO_POST_MORTEM = False
 
 
-
 def _initialDebugSetup(config):
     # do this part of debug setup first for easy debugging of import failures
     if config['debug']:
@@ -292,35 +274,22 @@ def _initialDebugSetup(config):
         defer.setDebugging(True)
 
 
-
-def _getSuites(config):
-    loader = _getLoader(config)
+def _getSuitesAndInputs(config):
+    #loader = irunner.TestLoader()
+    loader = runner.NetTestLoader()
     recurse = not config['no-recurse']
     print "loadByNames %s" % config['tests']
-    return loader.loadByNames(config['tests'], recurse)
-
-
-def _getLoader(config):
-    loader = runner.NetTestLoader()
-    if config['random']:
-        randomer = random.Random()
-        randomer.seed(config['random'])
-        loader.sorter = lambda x : randomer.random()
-        print 'Running tests shuffled with seed %d\n' % config['random']
-    return loader
-
+    inputs, suites = loader.loadByNamesWithInput(config['tests'], recurse)
+    return inputs, suites
 
 def _makeRunner(config):
     mode = None
     if config['debug']:
         mode = runner.OONIRunner.DEBUG
-    if config['dry-run']:
-        mode = runner.OONIRunner.DRY_RUN
     print "using %s" % config['reporter']
     return runner.OONIRunner(config['reporter'],
                               reportfile=config["reportfile"],
                               mode=mode,
-                              profile=config['profile'],
                               logfile=config['logfile'],
                               tracebackFormat=config['tbformat'],
                               realTimeErrors=config['rterrors'],
@@ -340,7 +309,8 @@ def run():
 
     _initialDebugSetup(config)
     trialRunner = _makeRunner(config)
-    suites = _getSuites(config)
-    for suite in suites:
-        test_result = trialRunner.run(suite)
+    inputs, testSuites = _getSuitesAndInputs(config)
+    log.startLogging(sys.stdout)
+    for i, suite in enumerate(testSuites):
+        test_result = trialRunner.run(suite, inputs[i])
 
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 07cffad..14297cd 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -89,7 +89,7 @@ class OONIReporter(OReporter):
 
     def getTestIndex(self, test):
         try:
-            idx = test.idx
+            idx = test._idx
         except:
             idx = 0
         return idx
@@ -109,7 +109,8 @@ class OONIReporter(OReporter):
         self._tests[idx]['input'] = test.input
         self._tests[idx]['idx'] = idx
         self._tests[idx]['name'] = test.name
-        self._tests[idx]['test'] = test
+        #self._tests[idx]['test'] = test
+        print "Now starting %s" % self._tests[idx]
 
 
     def stopTest(self, test):
diff --git a/ooni/runner.py b/ooni/runner.py
index 12ab9ad..604942d 100644
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@ -4,7 +4,7 @@ import types
 import time
 import inspect
 
-from twisted.internet import defer
+from twisted.internet import defer, reactor
 from twisted.python import reflect, log, failure
 from twisted.trial import unittest
 from twisted.trial.runner import TrialRunner, TestLoader
@@ -27,7 +27,7 @@ def isLegacyTest(obj):
     except TypeError:
         return False
 
-def adaptLegacyTest(obj):
+def adaptLegacyTest(obj, inputs=[None]):
     """
     We take a legacy OONITest class and convert it into a nettest.TestCase.
     This allows backward compatibility of old OONI tests.
@@ -36,8 +36,18 @@ def adaptLegacyTest(obj):
     older test cases compatible with the new OONI.
     """
     class LegacyOONITest(nettest.TestCase):
-        pass
+        inputs = [1]
+        original_test = obj
 
+        def test_start_legacy_test(self):
+            print "bla bla bla"
+            my_test = self.original_test()
+            print my_test
+            print "foobat"
+            my_test.startTest(self.input)
+            print "HHAHAHA"
+
+    return LegacyOONITest
 
 
 class LoggedSuite(nettest.TestSuite):
@@ -93,11 +103,14 @@ class OONISuite(nettest.TestSuite):
             self._bail()
 
 
-class NetTestLoader(object):
+class NetTestLoader(TestLoader):
     """
     Reponsible for finding the modules that can work as tests and running them.
     If we detect that a certain test is written using the legacy OONI API we
     will wrap it around a next gen class to make it work here too.
+
+    XXX This class needs to be cleaned up a *lot* of all the things we actually
+    don't need.
     """
     methodPrefix = 'test'
     modulePrefix = 'test_'
@@ -106,67 +119,19 @@ class NetTestLoader(object):
         self.suiteFactory = nettest.TestSuite
         self._importErrors = []
 
-
     def findTestClasses(self, module):
         classes = []
         for name, val in inspect.getmembers(module):
-            try:
-                inputs = val.inputs
-            except:
-                inputs = None
             if isTestCase(val):
-                classes.append((val, inputs))
+                classes.append(val)
             # This is here to allow backward compatibility with legacy OONI
             # tests.
             elif isLegacyTest(val):
-                #val = adaptLegacyTest(val)
-                classes.append((val, inputs))
+                print "adapting! %s" % val
+                val = adaptLegacyTest(val)
+                classes.append(val)
         return classes
 
-    def findByName(self, name):
-        """
-        Return a Python object given a string describing it.
-
-        @param name: a string which may be either a filename or a
-        fully-qualified Python name.
-
-        @return: If C{name} is a filename, return the module. If C{name} is a
-        fully-qualified Python name, return the object it refers to.
-        """
-        if os.path.exists(name):
-            return filenameToModule(name)
-        return reflect.namedAny(name)
-
-
-    def loadModule(self, module):
-        """
-        Return a test suite with all the tests from a module.
-
-        Included are TestCase subclasses and doctests listed in the module's
-        __doctests__ module. If that's not good for you, put a function named
-        either C{testSuite} or C{test_suite} in your module that returns a
-        TestSuite, and I'll use the results of that instead.
-
-        If C{testSuite} and C{test_suite} are both present, then I'll use
-        C{testSuite}.
-        """
-        ## XXX - should I add an optional parameter to disable the check for
-        ## a custom suite.
-        ## OR, should I add another method
-        if not isinstance(module, types.ModuleType):
-            raise TypeError("%r is not a module" % (module,))
-        if hasattr(module, 'testSuite'):
-            return module.testSuite()
-        elif hasattr(module, 'test_suite'):
-            return module.test_suite()
-
-        suite = self.suiteFactory()
-        for testClass, inputs in self.findTestClasses(module):
-            testCases = self.loadClass(testClass)
-
-        return testCases
-    loadTestsFromModule = loadModule
-
     def loadClass(self, klass):
         """
         Given a class which contains test cases, return a sorted list of
@@ -182,186 +147,50 @@ class NetTestLoader(object):
             tests.append(self._makeCase(klass, self.methodPrefix+name))
 
         suite = self.suiteFactory(tests)
-        suite.inputs = klass.inputs
-        return suite
-    loadTestsFromTestCase = loadClass
-
-    def getTestCaseNames(self, klass):
-        """
-        Given a class that contains C{TestCase}s, return a list of names of
-        methods that probably contain tests.
-        """
-        return reflect.prefixedMethodNames(klass, self.methodPrefix)
-
-    def loadMethod(self, method):
-        """
-        Given a method of a C{TestCase} that represents a test, return a
-        C{TestCase} instance for that test.
-        """
-        if not isinstance(method, types.MethodType):
-            raise TypeError("%r not a method" % (method,))
-        return self._makeCase(method.im_class, _getMethodNameInClass(method))
-
-    def _makeCase(self, klass, methodName):
-        return klass(methodName)
+        print "**+*"
+        print tests
+        print "**+*"
 
-    def loadPackage(self, package, recurse=False):
-        """
-        Load tests from a module object representing a package, and return a
-        TestSuite containing those tests.
-
-        Tests are only loaded from modules whose name begins with 'test_'
-        (or whatever C{modulePrefix} is set to).
-
-        @param package: a types.ModuleType object (or reasonable facsimilie
-        obtained by importing) which may contain tests.
-
-        @param recurse: A boolean.  If True, inspect modules within packages
-        within the given package (and so on), otherwise, only inspect modules
-        in the package itself.
-
-        @raise: TypeError if 'package' is not a package.
-
-        @return: a TestSuite created with my suiteFactory, containing all the
-        tests.
-        """
-        if not isPackage(package):
-            raise TypeError("%r is not a package" % (package,))
-        pkgobj = modules.getModule(package.__name__)
-        if recurse:
-            discovery = pkgobj.walkModules()
-        else:
-            discovery = pkgobj.iterModules()
-        discovered = []
-        for disco in discovery:
-            if disco.name.split(".")[-1].startswith(self.modulePrefix):
-                discovered.append(disco)
-        suite = self.suiteFactory()
-        for modinfo in self.sort(discovered):
-            try:
-                module = modinfo.load()
-            except:
-                thingToAdd = ErrorHolder(modinfo.name, failure.Failure())
-            else:
-                thingToAdd = self.loadModule(module)
-            suite.addTest(thingToAdd)
         return suite
+    loadTestsFromTestCase = loadClass
 
-    def loadDoctests(self, module):
-        """
-        Return a suite of tests for all the doctests defined in C{module}.
-
-        @param module: A module object or a module name.
-        """
-        if isinstance(module, str):
+    def findAllInputs(self, thing):
+        testClasses = self.findTestClasses(thing)
+        # XXX will there ever be more than 1 test class with inputs?
+        for klass in testClasses:
             try:
-                module = reflect.namedAny(module)
+                inputs = klass.inputs
             except:
-                return ErrorHolder(module, failure.Failure())
-        if not inspect.ismodule(module):
-            warnings.warn("trial only supports doctesting modules")
-            return
-        extraArgs = {}
-        if sys.version_info > (2, 4):
-            # Work around Python issue2604: DocTestCase.tearDown clobbers globs
-            def saveGlobals(test):
-                """
-                Save C{test.globs} and replace it with a copy so that if
-                necessary, the original will be available for the next test
-                run.
-                """
-                test._savedGlobals = getattr(test, '_savedGlobals', test.globs)
-                test.globs = test._savedGlobals.copy()
-            extraArgs['setUp'] = saveGlobals
-        return doctest.DocTestSuite(module, **extraArgs)
-
-    def loadAnything(self, thing, recurse=False):
-        """
-        Given a Python object, return whatever tests that are in it. Whatever
-        'in' might mean.
-
-        @param thing: A Python object. A module, method, class or package.
-        @param recurse: Whether or not to look in subpackages of packages.
-        Defaults to False.
-
-        @return: A C{TestCase} or C{TestSuite}.
-        """
-        print "Loading anything! %s" % thing
-        ret = None
-        if isinstance(thing, types.ModuleType):
-            if isPackage(thing):
-                ret = self.loadPackage(thing, recurse)
-            ret = self.loadModule(thing)
-        elif isinstance(thing, types.ClassType):
-            ret = self.loadClass(thing)
-        elif isinstance(thing, type):
-            ret = self.loadClass(thing)
-        elif isinstance(thing, types.MethodType):
-            ret = self.loadMethod(thing)
-        if not ret:
-            raise TypeError("No loader for %r. Unrecognized type" % (thing,))
-        try:
-            ret.inputs = ret.inputs
-        except:
-            ret.inputs = [None]
-        return ret
-
-    def loadByName(self, name, recurse=False):
-        """
-        Given a string representing a Python object, return whatever tests
-        are in that object.
-
-        If C{name} is somehow inaccessible (e.g. the module can't be imported,
-        there is no Python object with that name etc) then return an
-        L{ErrorHolder}.
-
-        @param name: The fully-qualified name of a Python object.
-        """
-        print "Load by Name!"
-        try:
-            thing = self.findByName(name)
-        except:
-            return ErrorHolder(name, failure.Failure())
-        return self.loadAnything(thing, recurse)
-    loadTestsFromName = loadByName
+                pass
+        return inputs
 
-    def loadByNames(self, names, recurse=False):
+    def loadByNamesWithInput(self, names, recurse=False):
         """
-        Construct a TestSuite containing all the tests found in 'names', where
+        Construct a OONITestSuite containing all the tests found in 'names', where
         names is a list of fully qualified python names and/or filenames. The
         suite returned will have no duplicate tests, even if the same object
         is named twice.
+
+        This test suite will have set the attribute inputs to the inputs found
+        inside of the tests.
         """
-        print "Load by Names!"
+        inputs = []
         things = []
         errors = []
         for name in names:
             try:
-                things.append(self.findByName(name))
+                thing = self.findByName(name)
+                things.append(thing)
             except:
                 errors.append(ErrorHolder(name, failure.Failure()))
-        suites = [self.loadAnything(thing, recurse)
-                  for thing in self._uniqueTests(things)]
-        suites.extend(errors)
-        return suites
-        #return self.suiteFactory(suites)
-
-
-    def _uniqueTests(self, things):
-        """
-        Gather unique suite objects from loaded things. This will guarantee
-        uniqueness of inherited methods on TestCases which would otherwise hash
-        to same value and collapse to one test unexpectedly if using simpler
-        means: e.g. set().
-        """
-        entries = []
-        for thing in things:
-            if isinstance(thing, types.MethodType):
-                entries.append((thing, thing.im_class))
-            else:
-                entries.append((thing,))
-        return [entry[0] for entry in set(entries)]
+        suites = []
+        for thing in self._uniqueTests(things):
+            inputs.append(self.findAllInputs(thing))
+            suite = self.loadAnything(thing, recurse)
+            suites.append(suite)
 
+        suites.extend(errors)
+        return inputs, suites
 
 class OONIRunner(object):
     """
@@ -457,47 +286,17 @@ class OONIRunner(object):
         self._logFileObserver = log.FileLogObserver(logFile)
         log.startLoggingWithObserver(self._logFileObserver.emit, 0)
 
-    def run(self, test):
+    def run(self, tests, inputs=[None]):
         """
         Run the test or suite and return a result object.
         """
-        print test
-        inputs = test.inputs
         reporterFactory = ReporterFactory(open(self._reportfile, 'a+'),
-                testSuite=test)
+                testSuite=tests)
         reporterFactory.writeHeader()
-        #testUnitReport = OONIReporter(open('reporting.log', 'a+'))
-        #testUnitReport.writeHeader(FooTest)
         for inputUnit in InputUnitFactory(inputs):
+            testSuiteFactory = nettest.TestSuiteFactory(inputUnit, tests, nettest.TestSuite)
             testUnitReport = reporterFactory.create()
-            test(testUnitReport, inputUnit)
+            for suite in testSuiteFactory:
+                suite(testUnitReport)
             testUnitReport.done()
 
-    def _runWithInput(self, test, input):
-        """
-        Private helper that runs the given test with the given input.
-        """
-        result = self._makeResult()
-        # decorate the suite with reactor cleanup and log starting
-        # This should move out of the runner and be presumed to be
-        # present
-        suite = TrialSuite([test])
-        startTime = time.time()
-
-        ## XXX replace this with the actual way of running the test.
-        run = lambda: suite.run(result)
-
-        oldDir = self._setUpTestdir()
-        try:
-            self._setUpLogFile()
-            run()
-        finally:
-            self._tearDownLogFile()
-            self._tearDownTestdir(oldDir)
-
-        endTime = time.time()
-        done = getattr(result, 'done', None)
-        result.done()
-        return result
-
-



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits