[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe
commit 2e49b7330b155ca4746896939ebe48ce67f147b3
Merge: 0080bf6 62956eb
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date: Thu Sep 13 23:16:03 2012 +0000
Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe
Conflicts:
ooni/assets/bridgetests.txt
ooni/ooniprobe.py
ooni/plugins/domclass.py
ooni/plugins/new_bridget.py
.gitmodules | 9 +
README.md | 22 ++-
TODO | 26 +--
ooni/__init__.py | 2 +-
ooni/assets/bridgetests.txt | 8 +
ooni/example_plugins/skel.py | 4 +-
ooni/lib/Makefile | 30 ---
ooni/lib/__init__.py | 41 +++-
ooni/lib/txscapy | 1 +
ooni/lib/txscapy.py | 363 -------------------------------
ooni/lib/txtorcon | 1 +
ooni/lib/txtraceroute | 1 +
ooni/ooniprobe.log | 1 -
ooni/ooniprobe.py | 19 ++-
ooni/oonitests/dnstamper.py | 142 ++++++++-----
ooni/plugins/blocking.py | 4 +-
ooni/plugins/dnstamper.py | 343 ++++++++++++++++++++++++++++++
ooni/plugins/domclass.py | 3 +-
ooni/plugins/httphost.py | 4 +-
ooni/plugins/new_bridget.py | 484 ++++++++++++++++++++++++++++++++++--------
ooni/plugoo/reports.py | 2 +-
ooni/plugoo/tests.py | 25 ++-
ooni/plugoo/work.py | 2 +
ooni/utils/log.py | 116 ++++++++---
24 files changed, 1045 insertions(+), 608 deletions(-)
diff --cc ooni/assets/bridgetests.txt
index 5519eea,7bba841..b69fbca
--- a/ooni/assets/bridgetests.txt
+++ b/ooni/assets/bridgetests.txt
@@@ -1,3 -1,5 +1,11 @@@
++<<<<<<< HEAD
+88.130.86.191:443
+195.74.237.236:9001
+127.0.0.1:9050
++=======
+ #213.151.89.102:9001
+ #108.166.106.156:443
+ #217.150.224.213:443
+ 85.193.252.111:443
+ #68.98.41.14:9001
++>>>>>>> 62956ebab7779c1b61ce3d6e8ac750552fd1c988
diff --cc ooni/plugins/new_bridget.py
index 3e4db56,0000000..736b084
mode 100644,000000..100644
--- a/ooni/plugins/new_bridget.py
+++ b/ooni/plugins/new_bridget.py
@@@ -1,105 -1,0 +1,421 @@@
- """
- This is a self genrated test created by scaffolding.py.
- you will need to fill it up with all your necessities.
- Safe hacking :).
- """
- from exceptions import Exception
- from datetime import datetime
- from zope.interface import implements
- from twisted.python import usage
- from twisted.plugin import IPlugin
- from twisted.internet import reactor, task
-
- from ooni.utils import log
- from ooni.plugoo.tests import ITest, OONITest
- from ooni.plugoo.assets import Asset
-
- from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState
- from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor
-
- class bridgetArgs(usage.Options):
- optParameters = [['bridges', 'b', None, 'List of bridges to scan'],
- ['relays', 'f', None, 'List of relays to scan'],
- ['resume', 'r', 0, 'Resume at this index'],
- ['timeout', 't', 5, 'Timeout in seconds after which to consider a bridge not working']
- ]
-
- class bridgetTest(OONITest):
++#!/usr/bin/env python
++# -*- encoding: utf-8 -*-
++#
++# +-----------+
++# | BRIDGET |
++# | +----------------------------------------------+
++# +--------| Use a slave Tor process to test making a Tor |
++# | connection to a list of bridges or relays. |
++# +----------------------------------------------+
++#
++# :authors: Arturo Filasto, Isis Lovecruft
++# :licence: see included LICENSE
++# :version: 0.1.0-alpha
++
++from __future__ import with_statement
++from zope.interface import implements
++from twisted.python import usage
++from twisted.plugin import IPlugin
++from twisted.internet import defer, error, reactor
++
++import random
++import sys
++
++try:
++ from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher
++except:
++ print "BridgeT requires txtorcon: https://github.com/meejah/txtorcon.git";
++ print "Your copy of OONI should have it included, if you're seeing this"
++ print "message, please file a bug report."
++ log.msg ("Bridget: Unable to import from ooni.lib.txtorcon")
++
++from ooni.utils import log
++from ooni.plugoo.tests import ITest, OONITest
++from ooni.plugoo.assets import Asset
++
++
++class BridgetArgs(usage.Options):
++
++ def portCheck(number):
++ number = int(number)
++ if number not in range(1024, 65535):
++ raise ValueError("Port out of range")
++ portCheck.coerceDoc = "Ports must be between 1024 and 65535"
++
++ optParameters = [
++ ['bridges', 'b', None,
++ 'List of bridges to scan <IP>:<ORport>'],
++ ['relays', 'f', None,
++ 'List of relays to scan <IP>'],
++ ['socks', 's', 9049, portCheck,
++ 'Tor SocksPort to use'],
++ ['control', 'c', 9052, portCheck,
++ 'Tor ControlPort to use'],
++ ['tor-path', 'p', None,
++ 'Path to the Tor binary to use'],
++ ['data-dir', 'd', None,
++ 'Tor DataDirectory to use'],
++ ['transport', 't', None,
++ 'Tor ClientTransportPlugin'],
++ ['resume', 'r', 0,
++ 'Resume at this index']]
++ optFlags = [['random', 'x', 'Randomize control and socks ports']]
++
++ def postOptions(self):
++ ## We can't test pluggable transports without bridges
++ if self['transport'] and not self['bridges']:
++ e = "Pluggable transport requires the bridges option"
++ raise usage.UsageError, e
++ ## We can't use random and static port simultaneously
++ if self['socks'] and self['control']:
++ if self['random']:
++ e = "Unable to use random and specific ports simultaneously"
++ raise usageError, e
++
++class CustomCircuit(CircuitListenerMixin):
++ implements(IStreamAttacher)
++
++ from txtorcon.interface import IRouterContainer, ICircuitContainer
++
++ def __init__(self, state):
++ self.state = state
++ self.waiting_circuits = []
++
++ def waiting_on(self, circuit):
++ for (circid, d) in self.waiting_circuits:
++ if circuit.id == circid:
++ return true
++ return False
++
++ def circuit_extend(self, circuit, router):
++ "ICircuitListener"
++ if circuit.purpose != 'GENERAL':
++ return
++ if self.waiting_on(circuit):
++ log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex))
++
++ def circuit_built(self, circuit):
++ "ICircuitListener"
++ if circuit.purpose != 'GENERAL':
++ return
++
++ log.msg("Circuit %s built ..." % circuit.id)
++ log.msg("Full path of %s: %s" % (circuit.id, circuit.path))
++
++ for (circid, d) in self.waiting_circuits:
++ if circid == circuit.id:
++ self.waiting_circuits.remove(circid, d)
++ d.callback(circuit)
++
++ def circuit_failed(self, circuit, reason):
++ if self.waiting_on(circuit):
++ log.msg("A circuit we requested %s failed for reason %s" %
++ (circuit.id, reason))
++ circid, d = None, None
++ for x in self.waiting_circuits:
++ if x[0] == circuit.id:
++ circid, d, stream_cc = x
++ if d is None:
++ raise Exception("Expected to find circuit.")
++
++ self.waiting_circuits.remove((circid, d))
++ log.msg("Trying to build a circuit for %s" % circid)
++ self.request_circuit_build(d)
++
++ def check_circuit_route(self, circuit, router):
++ if router in circuit.path:
++ #router.update() ## XXX can i use without args? no.
++ TorInfo.dump(self)
++
++ def request_circuit_build(self, deferred):
++ entries = self.state.entry_guards.value()
++ relays = self.state.routers.values()
++
++ log.msg("We have these nodes listed as entry guards:")
++ log.msg("%s" % entries)
++ log.msg("We have these nodes listed as relays:")
++ log.msg("%s" % relays)
++
++ path = [random.choice(entries),
++ random.choice(relays),
++ random.choice(relays)]
++
++ log.msg("Requesting a circuit: %s"
++ % '-->'.join(map(lambda x: x.location.countrycode, path)))
++
++ class AppendWaiting:
++ def __init__(self, attacher, deferred):
++ self.attacher = attacher
++ self.d = deferred
++
++ def __call__(self, circuit):
++ """
++ Return from build_circuit is a Circuit, however, we want to
++ wait until it is built before we can issue an attach on it and
++ callback to the Deferred we issue here.
++ """
++ log.msg("Circuit %s is in progress ..." % circuit.id)
++ self.attacher.waiting_circuits.append((circuit.id, self.d))
++
++ return self.state.build_circuit(path).addCallback(AppendWaiting(self, deferred_to_callback)).addErrback(log.err)
++
++class BridgetAsset(Asset):
++ """
++ Class for parsing bridge assets so that they can be commented out.
++ """
++ def __init__(self, file=None):
++ self = Asset.__init__(self, file)
++
++ def parse_line(self, line):
++ if line.startswith('#'):
++ return
++ else:
++ return line.replace('\n','')
++
++class BridgetTest(OONITest):
++ """
++ XXX fill me in
++
++ :ivar config:
++ An :class:`ooni.lib.txtorcon.TorConfig` instance.
++ :ivar relay_list:
++ A list of all provided relays to test. We have to do this because
++ txtorcon.TorState().entry_guards won't build a custom circuit if the
++ first hop isn't in the torrc's EntryNodes.
++ :ivar bridge_list:
++ A list of all provided bridges to test.
++ :ivar socks_port:
++ Integer for Tor's SocksPort.
++ :ivar control_port:
++ Integer for Tor's ControlPort.
++ :ivar plug_transport:
++ String defining the Tor's ClientTransportPlugin, for testing
++ a bridge's pluggable transport functionality.
++ :ivar tor_binary:
++ Path to the Tor binary to use, e.g. \'/usr/sbin/tor\'
++ """
+ implements(IPlugin, ITest)
+
+ shortName = "bridget"
- description = "bridget"
++ description = "Use a Tor process to test connecting to bridges and relays"
+ requirements = None
- options = bridgetArgs
++ options = BridgetArgs
+ blocking = False
+
- def experiment(self, args):
- log.msg("Doing test")
- last_update = datetime.now()
- tor_log = []
-
- def check_timeout():
- log.msg("Checking for timeout")
- time_since_update = datetime.now() - last_update
- if time_since_update.seconds > self.local_options['timeout']:
- log.msg("Timed out when connecting to %s" % args)
- l.stop()
- self.result['reason'] = 'timeout'
- d.errback(args)
- return
++ def initialize(self):
++ """
++ Extra initialization steps. We only want one child Tor process
++ running, so we need to deal with the creation of TorConfig() only
++ once, before the experiment runs.
++ """
++ self.socks_port = 9049
++ self.control_port = 9052
++ self.tor_binary = '/usr/sbin/tor'
++ self.data_directory = None
+
- def updates(prog, tag, summary):
- tor_log.append((prog, tag, summary))
- last_update = datetime.now()
- log.msg("%d%%: %s" % (prog, summary))
++ if self.local_options:
++ try:
++ from ooni.lib.txtorcon import TorConfig
++ except:
++ e = "Could not import TorConfig class from txtorcon!"
++ raise ImportError, e
+
- def setup_failed(failure):
- log.msg("Setup Failed.")
- if not self.result['reason']:
- self.result['reason'] = 'unknown'
- self.result['input'] = args
- self.result['result'] = 'failed'
- self.result['tor_log'] = tor_log
- return
++ options = self.local_options
++ self.config = TorConfig()
+
- def setup_complete(proto):
- log.msg("Setup Complete.")
- self.result['input'] = args
- self.result['result'] = 'success'
- return
++ ## Don't run the experiment if we don't have anything to test
++ if not options['bridges'] and not options['relays']:
++ self.suicide = True
++
++ if options['bridges']:
++ self.config.UseBridges = 1
+
- config = TorConfig()
- import random
- config.SocksPort = random.randint(1024, 2**16)
- config.ControlPort = random.randint(1024, 2**16)
++ if options['relays']:
++ ## Stupid hack for testing only relays:
++ ## Tor doesn't use EntryNodes when UseBridges is enabled, but
++ ## config.state.entry_guards needs to include the first hop to
++ ## build a custom circuit.
++ self.config.EntryNodes = ','.join(relay_list)
+
- if 'bridge' in args:
- config.UseBridges = 1
- config.Bridge = args['bridge']
++ if options['socks']:
++ self.socks_port = options['socks']
+
- config.save()
++ if options['control']:
++ self.control_port = options['control']
+
- print config.config
- self.result['tor_config'] = config.config
- log.msg("Starting Tor connecting to %s" % args['bridge'])
++ if options['random']:
++ log.msg("Using randomized ControlPort and SocksPort ...")
++ self.socks_port = random.randint(1024, 2**16)
++ self.control_port = random.randint(1024, 2**16)
+
- l = task.LoopingCall(check_timeout)
- l.start(1.0)
++ if options['tor-path']:
++ self.tor_binary = options['tor-path']
+
- d = launch_tor(config, self.reactor, control_port=config.ControlPort, progress_updates=updates)
- d.addCallback(setup_complete)
- d.addErrback(setup_failed)
- return d
++ if options['data-dir']:
++ self.config.DataDirectory = options['data-dir']
++
++ if options['transport']:
++ ## ClientTransportPlugin transport socks4|socks5 IP:PORT
++ ## ClientTransportPlugin transport exec path-to-binary [options]
++ if not options['bridges']:
++ e = "You must use the bridge option to test a transport."
++ raise usage.UsageError("%s" % e)
++
++ log.msg("Using pluggable transport ...")
++ ## XXX fixme there's got to be a better way to check the exec
++ assert type(options['transport']) is str
++ self.config.ClientTransportPlugin = options['transport']
++
++ self.config.SocksPort = self.socks_port
++ self.config.ControlPort = self.control_port
++ self.config.save()
+
+ def load_assets(self):
- assets = {}
++ """
++ Load bridges and/or relays from files given in user options. Bridges
++ should be given in the form IP:ORport. We don't want to load these as
++ assets, because it's inefficient to start a Tor process for each one.
++ """
++ assets = {}
++ self.bridge_list = []
++ self.relay_list = []
++
++ ## XXX fix me
++ ## we should probably find a more memory nice way to load addresses,
++ ## in case the files are really large
+ if self.local_options:
+ if self.local_options['bridges']:
- assets.update({'bridge': Asset(self.local_options['bridges'])})
- elif self.local_options['relays']:
- assets.update({'relay': Asset(self.local_options['relay'])})
++ log.msg("Loading bridge information from %s ..."
++ % self.local_options['bridges'])
++ with open(self.local_options['bridges']) as bridge_file:
++ for line in bridge_file.readlines():
++ if line.startswith('#'):
++ continue
++ else:
++ self.bridge_list.append(line.replace('\n',''))
++ assets.update({'bridges': self.bridge_list})
++
++ if self.local_options['relays']:
++ log.msg("Loading relay information from %s ..."
++ % self.local_options['relays'])
++ with open(options['relays']) as relay_file:
++ for line in relay_file.readlines():
++ if line.startswith('#'):
++ continue
++ else:
++ self.relay_list.append(line.replace('\n',''))
++ assets.update({'relays': self.relay_list})
+ return assets
+
- # We need to instantiate it otherwise getPlugins does not detect it
- # XXX Find a way to load plugins without instantiating them.
- bridget = bridgetTest(None, None, None)
++ def experiment(self, args):
++ """
++ XXX fill me in
++
++ :param args:
++ The :class:`ooni.plugoo.asset.Asset <Asset>` line currently being
++ used.
++ :meth launch_tor:
++ Returns a Deferred which callbacks with a
++ :class:`ooni.lib.txtorcon.torproto.TorProcessProtocol
++ <TorProcessProtocol>` connected to the fully-bootstrapped Tor;
++ this has a :class:`ooni.lib.txtorcon.torcontol.TorControlProtocol
++ <TorControlProtocol>` instance as .protocol.
++ """
++ from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState
++ from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor
++
++ def bootstrap(ctrl):
++ """
++ Launch a Tor process with the TorConfig instance returned from
++ initialize().
++ """
++ conf = TorConfig(ctrl)
++ conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail)
++ log.msg("Tor process connected, bootstrapping ...")
++
++ def reconf_controller(conf, bridge):
++ ## if bridges and relays, use one bridge then build a circuit
++ ## from three relays
++ conf.Bridge = bridge
++ ## XXX do we need a SIGHUP to restart?
++
++ ## XXX see txtorcon.TorControlProtocol.add_event_listener we
++ ## may not need full CustomCircuit class
++
++ ## if bridges only, try one bridge at a time, but don't build
++ ## circuits, just return
++ ## if relays only, build circuits from relays
++
++ def reconf_fail(args):
++ log.msg("Reconfiguring Tor config with args %s failed" % args)
++ reactor.stop()
++
++ def setup_fail(args):
++ log.msg("Setup Failed.")
++ report.update({'failed': args})
++ reactor.stop()
++
++ def setup_done(proto):
++ log.msg("Setup Complete: %s" % proto)
++ state = TorState(proto.tor_protocol)
++ state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail)
++ report.update({'success': args})
++
++ def updates(prog, tag, summary):
++ log.msg("%d%%: %s" % (prog, summary))
++
++ if len(args) == 0:
++ log.msg("Bridget needs lists of bridges and/or relays to test!")
++ log.msg("Exiting ...")
++ d = sys.exit()
++ return d
++
++ else:
++ if len(self.bridge_list) >= 1:
++ for bridge in self.bridge_list:
++ try:
++ print "BRIDGE IS %s" % bridge
++ reconf_controller(self.config, bridge)
++ except:
++ reconf_fail(bridge)
++
++ log.msg("Bridget: initiating test ... ")
++ log.msg("Using the following as our torrc:\n%s"
++ % self.config.create_torrc())
++ report = {'tor_config': self.config.config}
++ log.msg("Starting Tor ...")
++
++ ## :return: a Deferred which callbacks with a TorProcessProtocol
++ ## connected to the fully-bootstrapped Tor; this has a
++ ## txtorcon.TorControlProtocol instance as .protocol.
++ d = launch_tor(self.config,
++ reactor,
++ progress_updates=updates,
++ tor_binary=self.tor_binary)
++ d.addCallback(bootstrap, self.config)
++ d.addErrback(setup_fail)
++ ## now build circuits
++
++ #print "Tor process ID: %s" % d.transport.pid
++ return d
++
++## So that getPlugins() can register the Test:
++bridget = BridgetTest(None, None, None)
++
++## ISIS' NOTES
++## -----------
++## self.config.save() only needs to be called if Tor is already running.
++##
++## need to add transport type to torrc Bridge line:
++## Bridge <transport> IP:ORPort <fingerprint>
++##
++## TODO:
++## o add option for any kwarg=arg self.config setting
++## o cleanup documentation
++## o check if bridges are public relays
++## o take bridge_desc file as input, also be able to give same
++## format as output
++## o change the stupid name
++##
++## FIX:
++## data directory is not found, or permissions aren't right
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits