[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] * Still not sure what to do with the custodiet and bridget, also not sure



commit 0e45f922ca7293d1eceac76945385960407ad2ad
Author: Isis Lovecruft <isis@xxxxxxxxxxxxxx>
Date:   Sun Nov 4 12:24:42 2012 +0000

    * Still not sure what to do with the custodiet and bridget, also not sure
      about plugoo.
---
 nettests/bridget.py       |  499 +++++++++++++++++++++++++++++++++++++++++++++
 nettests/echo.py          |  196 ++++++++++++++++++
 nettests/tls-handshake.py |   32 +++
 ooni/custodiet.py         |  421 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 1148 insertions(+), 0 deletions(-)

diff --git a/nettests/bridget.py b/nettests/bridget.py
new file mode 100644
index 0000000..a334747
--- /dev/null
+++ b/nettests/bridget.py
@@ -0,0 +1,499 @@
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+#
+#  +-----------+
+#  |  BRIDGET  |
+#  |        +--------------------------------------------+
+#  +--------| Use a Tor process to test making a Tor     |
+#           | connection to a list of bridges or relays. |
+#           +--------------------------------------------+
+#
+# :authors: Isis Lovecruft, Arturo Filasto
+# :licence: see included LICENSE
+# :version: 0.1.0-alpha
+
+from __future__           import with_statement
+from functools            import partial
+from random               import randint
+
+import os
+import sys
+
+from twisted.python       import usage
+from twisted.plugin       import IPlugin
+from twisted.internet     import defer, error, reactor
+from zope.interface       import implements
+
+from ooni.utils           import log, date
+from ooni.utils.config    import ValueChecker
+
+from ooni.plugoo.tests    import ITest, OONITest
+from ooni.plugoo.assets   import Asset, MissingAssetException
+from ooni.utils.onion     import TxtorconImportError
+from ooni.utils.onion     import PTNoBridgesException, PTNotFoundException
+
+try:
+    from ooni.utils.onion     import parse_data_dir
+except:
+    log.msg("Please go to /ooni/lib and do 'make txtorcon' to run this test!")
+
+class RandomPortException(Exception):
+    """Raised when using a random port conflicts with configured ports."""
+    def __init__(self):
+        log.msg("Unable to use random and specific ports simultaneously")
+        return sys.exit()
+
+class BridgetArgs(usage.Options):
+    """Commandline options."""
+    allowed = "Port to use for Tor's %s, must be between 1024 and 65535."
+    sock_check = ValueChecker(allowed % "SocksPort").port_check
+    ctrl_check = ValueChecker(allowed % "ControlPort").port_check
+
+    optParameters = [
+        ['bridges', 'b', None,
+         'File listing bridge IP:ORPorts to test'],
+        ['relays', 'f', None,
+         'File listing relay IPs to test'],
+        ['socks', 's', 9049, None, sock_check],
+        ['control', 'c', 9052, None, ctrl_check],
+        ['torpath', 'p', None,
+         'Path to the Tor binary to use'],
+        ['datadir', 'd', None,
+         'Tor DataDirectory to use'],
+        ['transport', 't', None,
+         'Tor ClientTransportPlugin'],
+        ['resume', 'r', 0,
+         'Resume at this index']]
+    optFlags = [['random', 'x', 'Use random ControlPort and SocksPort']]
+
+    def postOptions(self):
+        if not self['bridges'] and not self['relays']:
+            raise MissingAssetException(
+                "Bridget can't run without bridges or relays to test!")
+        if self['transport']:
+            ValueChecker.uid_check(
+                "Can't run bridget as root with pluggable transports!")
+            if not self['bridges']:
+                raise PTNoBridgesException
+        if self['socks'] or self['control']:
+            if self['random']:
+                raise RandomPortException
+        if self['datadir']:
+            ValueChecker.dir_check(self['datadir'])
+        if self['torpath']:
+            ValueChecker.file_check(self['torpath'])
+
+class BridgetAsset(Asset):
+    """Class for parsing bridget Assets ignoring commented out lines."""
+    def __init__(self, file=None):
+        self = Asset.__init__(self, file)
+
+    def parse_line(self, line):
+        if line.startswith('#'):
+            return
+        else:
+            return line.replace('\n','')
+
+class BridgetTest(OONITest):
+    """
+    XXX fill me in
+
+    :ivar config:
+        An :class:`ooni.lib.txtorcon.TorConfig` instance.
+    :ivar relays:
+        A list of all provided relays to test.
+    :ivar bridges:
+        A list of all provided bridges to test.
+    :ivar socks_port:
+        Integer for Tor's SocksPort.
+    :ivar control_port:
+        Integer for Tor's ControlPort.
+    :ivar transport:
+        String defining the Tor's ClientTransportPlugin, for testing
+        a bridge's pluggable transport functionality.
+    :ivar tor_binary:
+        Path to the Tor binary to use, e.g. \'/usr/sbin/tor\'
+    """
+    implements(IPlugin, ITest)
+
+    shortName    = "bridget"
+    description  = "Use a Tor process to test connecting to bridges or relays"
+    requirements = None
+    options      = BridgetArgs
+    blocking     = False
+
+    def initialize(self):
+        """
+        Extra initialization steps. We only want one child Tor process
+        running, so we need to deal with most of the TorConfig() only once,
+        before the experiment runs.
+        """
+        self.socks_port      = 9049
+        self.control_port    = 9052
+        self.circuit_timeout = 90
+        self.tor_binary      = '/usr/sbin/tor'
+        self.data_directory  = None
+
+        def __make_asset_list__(opt, lst):
+            log.msg("Loading information from %s ..." % opt)
+            with open(opt) as opt_file:
+                for line in opt_file.readlines():
+                    if line.startswith('#'):
+                        continue
+                    else:
+                        lst.append(line.replace('\n',''))
+
+        def __count_remaining__(which):
+            total, reach, unreach = map(lambda x: which[x],
+                                        ['all', 'reachable', 'unreachable'])
+            count = len(total) - reach() - unreach()
+            return count
+
+        ## XXX should we do report['bridges_up'].append(self.bridges['current'])
+        self.bridges = {}
+        self.bridges['all'], self.bridges['up'], self.bridges['down'] = \
+            ([] for i in range(3))
+        self.bridges['reachable']   = lambda: len(self.bridges['up'])
+        self.bridges['unreachable'] = lambda: len(self.bridges['down'])
+        self.bridges['remaining']   = lambda: __count_remaining__(self.bridges)
+        self.bridges['current']     = None
+        self.bridges['pt_type']     = None
+        self.bridges['use_pt']      = False
+
+        self.relays = {}
+        self.relays['all'], self.relays['up'], self.relays['down'] = \
+            ([] for i in range(3))
+        self.relays['reachable']   = lambda: len(self.relays['up'])
+        self.relays['unreachable'] = lambda: len(self.relays['down'])
+        self.relays['remaining']   = lambda: __count_remaining__(self.relays)
+        self.relays['current']     = None
+
+        if self.local_options:
+            try:
+                from ooni.lib.txtorcon import TorConfig
+            except ImportError:
+                raise TxtorconImportError
+            else:
+                self.config = TorConfig()
+            finally:
+                options = self.local_options
+
+            if options['bridges']:
+                self.config.UseBridges = 1
+                __make_asset_list__(options['bridges'], self.bridges['all'])
+            if options['relays']:
+                ## first hop must be in TorState().guards
+                self.config.EntryNodes = ','.join(relay_list)
+                __make_asset_list__(options['relays'], self.relays['all'])
+            if options['socks']:
+                self.socks_port = options['socks']
+            if options['control']:
+                self.control_port = options['control']
+            if options['random']:
+                log.msg("Using randomized ControlPort and SocksPort ...")
+                self.socks_port   = randint(1024, 2**16)
+                self.control_port = randint(1024, 2**16)
+            if options['torpath']:
+                self.tor_binary = options['torpath']
+            if options['datadir']:
+                self.data_directory = parse_data_dir(options['datadir'])
+            if options['transport']:
+                ## ClientTransportPlugin transport exec pathtobinary [options]
+                ## XXX we need a better way to deal with all PTs
+                log.msg("Using ClientTransportPlugin %s" % options['transport'])
+                self.bridges['use_pt'] = True
+                [self.bridges['pt_type'], pt_exec] = \
+                    options['transport'].split(' ', 1)
+
+                if self.bridges['pt_type'] == "obfs2":
+                    self.config.ClientTransportPlugin = \
+                        self.bridges['pt_type'] + " " + pt_exec
+                else:
+                    raise PTNotFoundException
+
+            self.config.SocksPort            = self.socks_port
+            self.config.ControlPort          = self.control_port
+            self.config.CookieAuthentication = 1
+
+    def __load_assets__(self):
+        """
+        Load bridges and/or relays from files given in user options. Bridges
+        should be given in the form IP:ORport. We don't want to load these as
+        assets, because it's inefficient to start a Tor process for each one.
+
+        We cannot use the Asset model, because that model calls
+        self.experiment() with the current Assets, which would be one relay
+        and one bridge, then it gives the defer.Deferred returned from
+        self.experiment() to self.control(), which means that, for each
+        (bridge, relay) pair, experiment gets called again, which instantiates
+        an additional Tor process that attempts to bind to the same
+        ports. Thus, additionally instantiated Tor processes return with
+        RuntimeErrors, which break the final defer.chainDeferred.callback(),
+        sending it into the errback chain.
+        """
+        assets = {}
+        if self.local_options:
+            if self.local_options['bridges']:
+                assets.update({'bridge':
+                               BridgetAsset(self.local_options['bridges'])})
+            if self.local_options['relays']:
+                assets.update({'relay':
+                               BridgetAsset(self.local_options['relays'])})
+        return assets
+
+    def experiment(self, args):
+        """
+        if bridges:
+            1. configure first bridge line
+            2a. configure data_dir, if it doesn't exist
+            2b. write torrc to a tempfile in data_dir
+            3. start tor                              } if any of these
+            4. remove bridges which are public relays } fail, add current
+            5. SIGHUP for each bridge                 } bridge to unreach-
+                                                      } able bridges.
+        if relays:
+            1a. configure the data_dir, if it doesn't exist
+            1b. write torrc to a tempfile in data_dir
+            2. start tor
+            3. remove any of our relays which are already part of current
+               circuits
+            4a. attach CustomCircuit() to self.state
+            4b. RELAY_EXTEND for each relay } if this fails, add
+                                            } current relay to list
+                                            } of unreachable relays
+            5.
+        if bridges and relays:
+            1. configure first bridge line
+            2a. configure data_dir if it doesn't exist
+            2b. write torrc to a tempfile in data_dir
+            3. start tor
+            4. remove bridges which are public relays
+            5. remove any of our relays which are already part of current
+               circuits
+            6a. attach CustomCircuit() to self.state
+            6b. for each bridge, build three circuits, with three
+                relays each
+            6c. RELAY_EXTEND for each relay } if this fails, add
+                                            } current relay to list
+                                            } of unreachable relays
+
+        :param args:
+            The :class:`BridgetAsset` line currently being used. Except that it
+            in Bridget it doesn't, so it should be ignored and avoided.
+        """
+        try:
+            from ooni.utils         import process
+            from ooni.utils.onion   import remove_public_relays, start_tor
+            from ooni.utils.onion   import start_tor_filter_nodes
+            from ooni.utils.onion   import setup_fail, setup_done
+            from ooni.utils.onion   import CustomCircuit
+            from ooni.utils.timer   import deferred_timeout, TimeoutError
+            from ooni.lib.txtorcon  import TorConfig, TorState
+        except ImportError:
+            raise TxtorconImportError
+        except TxtorconImportError, tie:
+            log.err(tie)
+            sys.exit()
+
+        def reconfigure_done(state, bridges):
+            """
+            Append :ivar:`bridges['current']` to the list
+            :ivar:`bridges['up'].
+            """
+            log.msg("Reconfiguring with 'Bridge %s' successful"
+                    % bridges['current'])
+            bridges['up'].append(bridges['current'])
+            return state
+
+        def reconfigure_fail(state, bridges):
+            """
+            Append :ivar:`bridges['current']` to the list
+            :ivar:`bridges['down'].
+            """
+            log.msg("Reconfiguring TorConfig with parameters %s failed"
+                    % state)
+            bridges['down'].append(bridges['current'])
+            return state
+
+        @defer.inlineCallbacks
+        def reconfigure_bridge(state, bridges):
+            """
+            Rewrite the Bridge line in our torrc. If use of pluggable
+            transports was specified, rewrite the line as:
+                Bridge <transport_type> <IP>:<ORPort>
+            Otherwise, rewrite in the standard form:
+                Bridge <IP>:<ORPort>
+
+            :param state:
+                A fully bootstrapped instance of
+                :class:`ooni.lib.txtorcon.TorState`.
+            :param bridges:
+                A dictionary of bridges containing the following keys:
+
+                bridges['remaining'] :: A function returning and int for the
+                                        number of remaining bridges to test.
+                bridges['current']   :: A string containing the <IP>:<ORPort>
+                                        of the current bridge.
+                bridges['use_pt']    :: A boolean, True if we're testing
+                                        bridges with a pluggable transport;
+                                        False otherwise.
+                bridges['pt_type']   :: If :ivar:`bridges['use_pt'] is True,
+                                        this is a string containing the type
+                                        of pluggable transport to test.
+            :return:
+                :param:`state`
+            """
+            log.msg("Current Bridge: %s" % bridges['current'])
+            log.msg("We now have %d bridges remaining to test..."
+                    % bridges['remaining']())
+            try:
+                if bridges['use_pt'] is False:
+                    controller_response = yield state.protocol.set_conf(
+                        'Bridge', bridges['current'])
+                elif bridges['use_pt'] and bridges['pt_type'] is not None:
+                    controller_reponse = yield state.protocol.set_conf(
+                        'Bridge', bridges['pt_type'] +' '+ bridges['current'])
+                else:
+                    raise PTNotFoundException
+
+                if controller_response == 'OK':
+                    finish = yield reconfigure_done(state, bridges)
+                else:
+                    log.err("SETCONF for %s responded with error:\n %s"
+                            % (bridges['current'], controller_response))
+                    finish = yield reconfigure_fail(state, bridges)
+
+                defer.returnValue(finish)
+
+            except Exception, e:
+                log.err("Reconfiguring torrc with Bridge line %s failed:\n%s"
+                        % (bridges['current'], e))
+                defer.returnValue(None)
+
+        def attacher_extend_circuit(attacher, deferred, router):
+            ## XXX todo write me
+            ## state.attacher.extend_circuit
+            raise NotImplemented
+            #attacher.extend_circuit
+
+        def state_attach(state, path):
+            log.msg("Setting up custom circuit builder...")
+            attacher = CustomCircuit(state)
+            state.set_attacher(attacher, reactor)
+            state.add_circuit_listener(attacher)
+            return state
+
+            ## OLD
+            #for circ in state.circuits.values():
+            #    for relay in circ.path:
+            #        try:
+            #            relay_list.remove(relay)
+            #        except KeyError:
+            #            continue
+            ## XXX how do we attach to circuits with bridges?
+            d = defer.Deferred()
+            attacher.request_circuit_build(d)
+            return d
+
+        def state_attach_fail(state):
+            log.err("Attaching custom circuit builder failed: %s" % state)
+
+        log.msg("Bridget: initiating test ... ")  ## Start the experiment
+
+        ## if we've at least one bridge, and our config has no 'Bridge' line
+        if self.bridges['remaining']() >= 1 \
+                and not 'Bridge' in self.config.config:
+
+            ## configure our first bridge line
+            self.bridges['current'] = self.bridges['all'][0]
+            self.config.Bridge = self.bridges['current']
+                                                  ## avoid starting several
+            self.config.save()                    ## processes
+            assert self.config.config.has_key('Bridge'), "No Bridge Line"
+
+            ## start tor and remove bridges which are public relays
+            from ooni.utils.onion import start_tor_filter_nodes
+            state = start_tor_filter_nodes(reactor, self.config,
+                                           self.control_port, self.tor_binary,
+                                           self.data_directory, self.bridges)
+            #controller = defer.Deferred()
+            #controller.addCallback(singleton_semaphore, tor)
+            #controller.addErrback(setup_fail)
+            #bootstrap = defer.gatherResults([controller, filter_bridges],
+            #                                consumeErrors=True)
+
+            if state is not None:
+                log.debug("state:\n%s" % state)
+                log.debug("Current callbacks on TorState():\n%s"
+                          % state.callbacks)
+
+        ## if we've got more bridges
+        if self.bridges['remaining']() >= 2:
+            #all = []
+            for bridge in self.bridges['all'][1:]:
+                self.bridges['current'] = bridge
+                #new = defer.Deferred()
+                #new.addCallback(reconfigure_bridge, state, self.bridges)
+                #all.append(new)
+            #check_remaining = defer.DeferredList(all, consumeErrors=True)
+            #state.chainDeferred(check_remaining)
+                state.addCallback(reconfigure_bridge, self.bridges)
+
+        if self.relays['remaining']() > 0:
+            while self.relays['remaining']() >= 3:
+                #path = list(self.relays.pop() for i in range(3))
+                #log.msg("Trying path %s" % '->'.join(map(lambda node:
+                #                                         node, path)))
+                self.relays['current'] = self.relays['all'].pop()
+                for circ in state.circuits.values():
+                    for node in circ.path:
+                        if node == self.relays['current']:
+                            self.relays['up'].append(self.relays['current'])
+                    if len(circ.path) < 3:
+                        try:
+                            ext = attacher_extend_circuit(state.attacher, circ,
+                                                          self.relays['current'])
+                            ext.addCallback(attacher_extend_circuit_done,
+                                            state.attacher, circ,
+                                            self.relays['current'])
+                        except Exception, e:
+                            log.err("Extend circuit failed: %s" % e)
+                    else:
+                        continue
+
+        #state.callback(all)
+        #self.reactor.run()
+        return state
+
+    def startTest(self, args):
+        """
+        Local override of :meth:`OONITest.startTest` to bypass calling
+        self.control.
+
+        :param args:
+            The current line of :class:`Asset`, not used but kept for
+            compatibility reasons.
+        :return:
+            A fired deferred which callbacks :meth:`experiment` and
+            :meth:`OONITest.finished`.
+        """
+        self.start_time = date.now()
+        self.d = self.experiment(args)
+        self.d.addErrback(log.err)
+        self.d.addCallbacks(self.finished, log.err)
+        return self.d
+
+## So that getPlugins() can register the Test:
+#bridget = BridgetTest(None, None, None)
+
+## ISIS' NOTES
+## -----------
+## TODO:
+##       x  cleanup documentation
+##       x  add DataDirectory option
+##       x  check if bridges are public relays
+##       o  take bridge_desc file as input, also be able to give same
+##          format as output
+##       x  Add asynchronous timeout for deferred, so that we don't wait
+##       o  Add assychronous timout for deferred, so that we don't wait
+##          forever for bridges that don't work.
diff --git a/nettests/echo.py b/nettests/echo.py
new file mode 100644
index 0000000..bc47519
--- /dev/null
+++ b/nettests/echo.py
@@ -0,0 +1,196 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+#  +---------+
+#  | echo.py |
+#  +---------+
+#     A simple ICMP-8 ping test.
+#
+# :author: Isis Lovecruft
+# :version: 0.0.1-pre-alpha
+# :license: (c) 2012 Isis Lovecruft
+#           see attached LICENCE file
+#
+
+import os
+import sys
+
+from pprint           import pprint
+
+from twisted.internet import reactor
+from twisted.plugin   import IPlugin
+from twisted.python   import usage
+from ooni.nettest     import NetTestCase
+from ooni.utils       import log, Storage
+from ooni.utils.net   import PermissionsError, IfaceError
+
+try:
+    from scapy.all             import sr1, IP, ICMP        ## XXX v4/v6?
+    from ooni.lib              import txscapy
+    from ooni.lib.txscapy      import txsr, txsend
+    from ooni.templates.scapyt import ScapyTest
+except:
+    log.msg("This test requires scapy, see www.secdev.org/projects/scapy")
+
+## xxx TODO: move these to a utility function for determining OSes
+LINUX=sys.platform.startswith("linux")
+OPENBSD=sys.platform.startswith("openbsd")
+FREEBSD=sys.platform.startswith("freebsd")
+NETBSD=sys.platform.startswith("netbsd")
+DARWIN=sys.platform.startswith("darwin")
+SOLARIS=sys.platform.startswith("sunos")
+WINDOWS=sys.platform.startswith("win32")
+
+class EchoTest(ScapyTest):
+    """
+    xxx fill me in
+    """
+    name         = 'echo'
+    author       = 'Isis Lovecruft <isis@xxxxxxxxxxxxxx>'
+    description  = 'A simple ICMP-8 test to see if a host is reachable.'
+    version      = '0.0.1'
+    inputFile    = ['file', 'f', None, 'File of list of IPs to ping']
+    requirements = None
+    #report       = Storage()
+
+    optParameters = [
+        ['interface', 'i', None, 'Network interface to use'],
+        ['count', 'c', 5, 'Number of packets to send', int],
+        ['size', 's', 56, 'Number of bytes to send in ICMP data field', int],
+        ['ttl', 'l', 25, 'Set the IP Time to Live', int],
+        ['timeout', 't', 2, 'Seconds until timeout if no response', int],
+        ['pcap', 'p', None, 'Save pcap to this file'],
+        ['receive', 'r', True, 'Receive response packets']
+        ]
+
+    def setUp(self, *a, **kw):
+        '''
+        :ivar ifaces:
+            Struct returned from getifaddrs(3) and turned into a tuple in the
+            form (*ifa_name, AF_FAMILY, *ifa_addr)
+        '''
+
+        if self.local_options:
+            log.debug("%s: local_options found" % self.name)
+            for key, value in self.local_options:
+                log.debug("%s: setting self.%s = %s" % (key, value))
+                setattr(self, key, value)
+
+        ## xxx is this now .subOptions?
+        #self.inputFile = self.localOptions['file']
+        self.timeout *= 1000            ## convert to milliseconds
+
+        if not self.interface:
+            log.msg("No network interface specified!")
+            log.debug("OS detected: %s" % sys.platform)
+            if LINUX or OPENBSD or NETBSD or FREEBSD or DARWIN or SOLARIS:
+                from twisted.internet.test import _posixifaces
+                log.msg("Attempting to discover network interfaces...")
+                ifaces = _posixifaces._interfaces()
+            elif WINDOWS:
+                from twisted.internet.test import _win32ifaces
+                log.msg("Attempting to discover network interfaces...")
+                ifaces = _win32ifaces._interfaces()
+            else:
+                log.debug("Client OS %s not accounted for!" % sys.platform)
+                log.debug("Unable to discover network interfaces...")
+                ifaces = [('lo', '')]
+
+            ## found = {'eth0': '1.1.1.1'}
+            found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo']
+            log.info("Found interfaces:\n%s" % pprint(found))
+            self.interfaces = self.tryInterfaces(found)
+        else:
+            ## xxx need a way to check that iface exists, is up, and
+            ## we have permissions on it
+            log.debug("Our interface has been set to %s" % self.interface)
+
+        if self.pcap:
+            try:
+                self.pcapfile = open(self.pcap, 'a+')
+            except:
+                log.msg("Unable to write to pcap file %s" % self.pcap)
+                self.pcapfile = None
+
+        try:
+            assert os.path.isfile(self.file)
+            fp = open(self.file, 'r')
+        except Exception, e:
+            hosts = ['8.8.8.8', '38.229.72.14']
+            log.err(e)
+        else:
+            self.inputs = self.inputProcessor(fp)
+        self.removePorts(hosts)
+
+        log.debug("Initialization of %s test completed with:\n%s"
+                  % (self.name, ''.join(self.__dict__)))
+
+    @staticmethod
+    def inputParser(inputs):
+        log.debug("Removing possible ports from host addresses...")
+        log.debug("Initial inputs:\n%s" % pprint(inputs))
+
+        assert isinstance(inputs, list)
+        hosts = [h.rsplit(':', 1)[0] for h in inputs]
+        log.debug("Inputs converted to:\n%s" % hosts)
+
+        return hosts
+
+    def tryInterfaces(self, ifaces):
+        try:
+            from scapy.all import sr1   ## we want this check to be blocking
+        except:
+            log.msg("This test requires scapy: www.secdev.org/projects/scapy")
+            raise SystemExit
+
+        ifup = {}
+        while ifaces:
+            for ifname, ifaddr in ifaces:
+                log.debug("Currently testing network capabilities of interface"
+                          + "%s  by sending a packet to our address %s"
+                          % (ifname, ifaddr))
+                try:
+                    pkt = IP(dst=ifaddr)/ICMP()
+                    ans, unans = sr(pkt, iface=ifname, timeout=self.timeout)
+                except Exception, e:
+                    raise PermissionsError if e.find("Errno 1") else log.err(e)
+                else:
+                    ## xxx i think this logic might be wrong
+                    log.debug("Interface test packet\n%s\n\n%s"
+                              % (pkt.summary(), pkt.show2()))
+                    if ans.summary():
+                        log.info("Received answer for test packet on interface"
+                                 +"%s :\n%s" % (ifname, ans.summary()))
+                        ifup.update(ifname, ifaddr)
+                    else:
+                        log.info("Our interface test packet was unanswered:\n%s"
+                                 % unans.summary())
+
+        if len(ifup) > 0:
+            log.msg("Discovered the following working network interfaces: %s"
+                    % ifup)
+            return ifup
+        else:
+            raise IfaceError("Could not find a working network interface.")
+
+    def buildPackets(self):
+        log.debug("self.input is %s" % self.input)
+        log.debug("self.hosts is %s" % self.hosts)
+        for addr in self.input:
+            packet = IP(dst=self.input)/ICMP()
+            self.request.append(packet)
+        return packet
+
+    def test_icmp(self):
+        if self.recieve:
+            self.buildPackets()
+            all = []
+            for packet in self.request:
+                d = self.sendReceivePackets(packets=packet)
+                all.append(d)
+                self.response.update({packet: d})
+            d_list = defer.DeferredList(all)
+            return d_list
+        else:
+            d = self.sendPackets()
+            return d
diff --git a/nettests/tls-handshake.py b/nettests/tls-handshake.py
new file mode 100644
index 0000000..eba950e
--- /dev/null
+++ b/nettests/tls-handshake.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+import subprocess
+from subprocess import PIPE
+serverport = "129.21.124.215:443"
+# a subset of those from firefox
+ciphers = [
+  "ECDHE-ECDSA-AES256-SHA",
+  "ECDHE-RSA-AES256-SHA",
+  "DHE-RSA-CAMELLIA256-SHA",
+  "DHE-DSS-CAMELLIA256-SHA",
+  "DHE-RSA-AES256-SHA",
+  "DHE-DSS-AES256-SHA",
+  "ECDH-ECDSA-AES256-CBC-SHA",
+  "ECDH-RSA-AES256-CBC-SHA",
+  "CAMELLIA256-SHA",
+  "AES256-SHA",
+  "ECDHE-ECDSA-RC4-SHA",
+  "ECDHE-ECDSA-AES128-SHA",
+  "ECDHE-RSA-RC4-SHA",
+  "ECDHE-RSA-AES128-SHA",
+  "DHE-RSA-CAMELLIA128-SHA",
+  "DHE-DSS-CAMELLIA128-SHA"
+]
+def checkBridgeConnection(host, port)
+  cipher_arg = ":".join(ciphers)
+  cmd  = ["openssl", "s_client", "-connect", "%s:%s" % (host,port)]
+  cmd += ["-cipher", cipher_arg]
+  proc = subprocess.Popen(cmd, stdout=PIPE, stderr=PIPE,stdin=PIPE)
+  out, error = proc.communicate()
+  success = "Cipher is DHE-RSA-AES256-SHA" in out
+  return success
diff --git a/ooni/custodiet.py b/ooni/custodiet.py
new file mode 100755
index 0000000..8cbcfce
--- /dev/null
+++ b/ooni/custodiet.py
@@ -0,0 +1,421 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8
+#
+#    custodiet
+#    *********
+#
+#    "...quis custodiet ipsos custodes?"
+#               - Juvenal, Satires VI.347-348 (circa 2nd Century, C.E.)
+#
+#        "'Hand me the Custodian,' Goodchild demands, inserting the waiflike
+#    robot into Bambara's opened navel. 'Providing conscience for those who
+#    have none.' Goodchild and the other Breen government agents disappear
+#    into the surrounding desert in a vehicle, kicking up cloud of white dust.
+#        Bambara awakens, and, patting the dust from his clothing, turns to
+#    greet a one-armed child. 'Hi, my name's Bambara; I'm a
+#    thirty-six-year-old Virgo and a former killer, who's hobbies include
+#    performing recreational autopsies, defecating, and drinking rum. I've
+#    recently been given a conscience, and would very much like to help you.'
+#        Cut to Bambara and the child, now with one of Bambara's arms, leaving
+#    a surgical clinic."
+#               - AeonFlux, "The Purge" (sometime in the late 90s)
+#
+#    :copyright: (c) 2012 Isis Lovecruft
+#    :license: see LICENSE for more details.
+#    :version: 0.1.0-beta
+#
+
+# ooniprobe.py imports
+import sys
+from signal import SIGTERM, signal
+from pprint import pprint
+
+from twisted.python import usage
+from twisted.internet import reactor
+from twisted.plugin import getPlugins
+
+from zope.interface.verify import verifyObject
+from zope.interface.exceptions import BrokenImplementation
+from zope.interface.exceptions import BrokenMethodImplementation
+
+from ooni.bridget.tests import bridget
+from ooni.bridget.utils import log, tests, work, reports
+from ooni.bridget.utils.interface import ITest
+from ooni.utils.logo import getlogo
+
+# runner.py imports
+import os
+import types
+import time
+import inspect
+import yaml
+
+from twisted.internet import defer, reactor
+from twisted.python   import reflect, failure, usage
+from twisted.python   import log as tlog
+
+from twisted.trial        import unittest
+from twisted.trial.runner import TrialRunner, TestLoader
+from twisted.trial.runner import isPackage, isTestCase, ErrorHolder
+from twisted.trial.runner import filenameToModule, _importFromFile
+
+from ooni              import nettest
+from ooni.inputunit    import InputUnitFactory
+from ooni.nettest      import InputTestSuite
+from ooni.plugoo       import tests as oonitests
+from ooni.reporter     import ReporterFactory
+from ooni.utils        import log, geodata, date
+from ooni.utils.legacy import LegacyOONITest
+from ooni.utils.legacy import start_legacy_test, adapt_legacy_test
+
+
+__version__ = "0.1.0-beta"
+
+
+#def retrieve_plugoo():
+#    """
+#    Get all the plugins that implement the ITest interface and get the data
+#    associated to them into a dict.
+#    """
+#    interface = ITest
+#    d = {}
+#    error = False
+#    for p in getPlugins(interface, plugins):
+#        try:
+#            verifyObject(interface, p)
+#            d[p.shortName] = p
+#        except BrokenImplementation, bi:
+#            print "Plugin Broken"
+#            print bi
+#            error = True
+#    if error != False:
+#        print "Plugin Loaded!"
+#    return d
+#
+#plugoo = retrieve_plugoo()
+
+"""
+
+ai to watch over which tests to run - custodiet
+
+   * runTest() or getPrefixMethodNames() to run the tests in order for each
+     test (esp. the tcp and icmp parts) to be oonicompat we should use the
+     test_icmp_ping API framework for those.
+
+   * should handle calling
+
+tests to run:
+  echo
+  syn
+  fin
+  conn
+  tls
+  tor
+need fakebridge - canary
+
+"""
+
+def runTest(test, options, global_options, reactor=reactor):
+    """
+    Run an OONI probe test by name.
+
+    @param test: a string specifying the test name as specified inside of
+                 shortName.
+
+    @param options: the local options to be passed to the test.
+
+    @param global_options: the global options for OONI
+    """
+    parallelism = int(global_options['parallelism'])
+    worker = work.Worker(parallelism, reactor=reactor)
+    test_class = plugoo[test].__class__
+    report = reports.Report(test, global_options['output'])
+
+    log_to_stdout = True
+    if global_options['quiet']:
+        log_to_stdout = False
+
+    log.start(log_to_stdout,
+              global_options['log'],
+              global_options['verbosity'])
+
+    resume = 0
+    if not options:
+        options = {}
+    if 'resume' in options:
+        resume = options['resume']
+
+    test = test_class(options, global_options, report, reactor=reactor)
+    if test.tool:
+        test.runTool()
+        return True
+
+    if test.ended:
+        print "Ending test"
+        return None
+
+    wgen = work.WorkGenerator(test,
+                              dict(options),
+                              start=resume)
+    for x in wgen:
+        worker.push(x)
+
+class MainOptions(usage.Options):
+    tests = [bridget, ]
+    subCommands = []
+    for test in tests:
+        print test
+        testopt = getattr(test, 'options')
+        subCommands.append([test, None, testopt, "Run the %s test" % test])
+
+    optFlags = [
+        ['quiet', 'q', "Don't log to stdout"]
+    ]
+
+    optParameters = [
+        ['parallelism', 'n', 10, "Specify the number of parallel tests to run"],
+        #['target-node', 't', 'localhost:31415', 'Select target node'],
+        ['output', 'o', 'bridge.log', "Specify output report file"],
+        ['reportfile', 'o', 'bridge.log', "Specify output log file"],
+        ['verbosity', 'v', 1, "Specify the logging level"],
+    ]
+
+    def opt_version(self):
+        """
+        Display OONI version and exit.
+        """
+        print "OONI version:", __version__
+        sys.exit(0)
+
+    def __str__(self):
+        """
+        Hack to get the sweet ascii art into the help output and replace the
+        strings "Commands" with "Tests".
+        """
+        return getlogo() + '\n' + self.getSynopsis() + '\n' + \
+               self.getUsage(width=None).replace("Commands:", "Tests:")
+
+
+
+def isTestCase(thing):
+    try:
+        return issubclass(thing, unittest.TestCase)
+    except TypeError:
+        return False
+
+def isLegacyTest(obj):
+    """
+    Returns True if the test in question is written using the OONITest legacy
+    class.
+    We do this for backward compatibility of the OONIProbe API.
+    """
+    try:
+        if issubclass(obj, oonitests.OONITest) and not obj == oonitests.OONITest:
+                return True
+        else:
+            return False
+    except TypeError:
+        return False
+
+def processTest(obj, config):
+    """
+    Process the parameters and :class:`twisted.python.usage.Options` of a
+    :class:`ooni.nettest.Nettest`.
+
+    :param obj:
+        An uninstantiated old test, which should be a subclass of
+        :class:`ooni.plugoo.tests.OONITest`.
+    :param config:
+        A configured and instantiated :class:`twisted.python.usage.Options`
+        class.
+    """
+
+    inputFile = obj.inputFile
+
+    if obj.optParameters or inputFile:
+        if not obj.optParameters:
+            obj.optParameters = []
+
+        if inputFile:
+            obj.optParameters.append(inputFile)
+
+        class Options(usage.Options):
+            optParameters = obj.optParameters
+
+        options = Options()
+        options.parseOptions(config['subArgs'])
+        obj.localOptions = options
+
+        if inputFile:
+            obj.inputFile = options[inputFile[0]]
+        try:
+            tmp_obj = obj()
+            tmp_obj.getOptions()
+        except usage.UsageError:
+            options.opt_help()
+
+    return obj
+
+def findTestClassesFromConfig(config):
+    """
+    Takes as input the command line config parameters and returns the test
+    case classes.
+    If it detects that a certain test class is using the old OONIProbe format,
+    then it will adapt it to the new testing system.
+
+    :param config:
+        A configured and instantiated :class:`twisted.python.usage.Options`
+        class.
+    :return:
+        A list of class objects found in a file or module given on the
+        commandline.
+    """
+
+    filename = config['test']
+    classes = []
+
+    module = filenameToModule(filename)
+    for name, val in inspect.getmembers(module):
+        if isTestCase(val):
+            classes.append(processTest(val, config))
+        elif isLegacyTest(val):
+            classes.append(adapt_legacy_test(val, config))
+    return classes
+
+def makeTestCases(klass, tests, methodPrefix):
+    """
+    Takes a class some tests and returns the test cases. methodPrefix is how
+    the test case functions should be prefixed with.
+    """
+
+    cases = []
+    for test in tests:
+        cases.append(klass(methodPrefix+test))
+    return cases
+
+def loadTestsAndOptions(classes, config):
+    """
+    Takes a list of classes and returns their testcases and options.
+    Legacy tests will be adapted.
+    """
+
+    methodPrefix = 'test'
+    suiteFactory = InputTestSuite
+    options = []
+    testCases = []
+    names = []
+
+    _old_klass_type = LegacyOONITest
+
+    for klass in classes:
+        if isinstance(klass, _old_klass_type):
+            try:
+                cases = start_legacy_test(klass)
+                #cases.callback()
+                if cases:
+                    print cases
+                    return [], []
+                testCases.append(cases)
+            except Exception, e:
+                log.err(e)
+            else:
+                try:
+                    opts = klass.local_options
+                    options.append(opts)
+                except AttributeError, ae:
+                    options.append([])
+                    log.err(ae)
+        elif not isinstance(klass, _old_klass_type):
+            tests = reflect.prefixedMethodNames(klass, methodPrefix)
+            if tests:
+                cases = makeTestCases(klass, tests, methodPrefix)
+                testCases.append(cases)
+            try:
+                k = klass()
+                opts = k.getOptions()
+                options.append(opts)
+            except AttributeError, ae:
+                options.append([])
+                log.err(ae)
+        else:
+            try:
+                raise RuntimeError, "Class is some strange type!"
+            except RuntimeError, re:
+                log.err(re)
+
+    return testCases, options
+
+class ORunner(object):
+    """
+    This is a specialized runner used by the ooniprobe command line tool.
+    I am responsible for reading the inputs from the test files and splitting
+    them in input units. I also create all the report instances required to run
+    the tests.
+    """
+    def __init__(self, cases, options=None, config=None, *arg, **kw):
+        self.baseSuite = InputTestSuite
+        self.cases = cases
+        self.options = options
+
+        try:
+            assert len(options) != 0, "Length of options is zero!"
+        except AssertionError, ae:
+            self.inputs = []
+            log.err(ae)
+        else:
+            try:
+                first = options.pop(0)
+            except:
+                first = {}
+            if 'inputs' in first:
+                self.inputs = options['inputs']
+            else:
+                log.msg("Could not find inputs!")
+                log.msg("options[0] = %s" % first)
+                self.inputs = [None]
+
+        try:
+            reportFile = open(config['reportfile'], 'a+')
+        except:
+            filename = 'report_'+date.timestamp()+'.yaml'
+            reportFile = open(filename, 'a+')
+        self.reporterFactory = ReporterFactory(reportFile,
+                                               testSuite=self.baseSuite(self.cases))
+
+    def runWithInputUnit(self, inputUnit):
+        idx = 0
+        result = self.reporterFactory.create()
+
+        for inputs in inputUnit:
+            result.reporterFactory = self.reporterFactory
+
+            suite = self.baseSuite(self.cases)
+            suite.input = inputs
+            suite(result, idx)
+
+            # XXX refactor all of this index bullshit to avoid having to pass
+            # this index around. Probably what I want to do is go and make
+            # changes to report to support the concept of having multiple runs
+            # of the same test.
+            # We currently need to do this addition in order to get the number
+            # of times the test cases that have run inside of the test suite.
+            idx += (suite._idx - idx)
+
+        result.done()
+
+    def run(self):
+        self.reporterFactory.options = self.options
+        for inputUnit in InputUnitFactory(self.inputs):
+            self.runWithInputUnit(inputUnit)
+
+if __name__ == "__main__":
+    config = Options()
+    config.parseOptions()
+
+    if not config.subCommand:
+        config.opt_help()
+        signal(SIGTERM)
+        #sys.exit(1)
+
+    runTest(config.subCommand, config.subOptions, config)
+    reactor.run()



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits