[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] r14542: Backport documentation to trunk. (in torflow/trunk: . TorCtl)
Author: mikeperry
Date: 2008-05-03 05:30:18 -0400 (Sat, 03 May 2008)
New Revision: 14542
Modified:
torflow/trunk/README
torflow/trunk/TorCtl/PathSupport.py
torflow/trunk/TorCtl/README
torflow/trunk/TorCtl/TorCtl.py
torflow/trunk/TorCtl/TorUtil.py
torflow/trunk/metatroller.py
Log:
Backport documentation to trunk.
Modified: torflow/trunk/README
===================================================================
--- torflow/trunk/README 2008-05-03 09:28:43 UTC (rev 14541)
+++ torflow/trunk/README 2008-05-03 09:30:18 UTC (rev 14542)
@@ -26,7 +26,14 @@
circuit construction and stream attachment subject to policies defined
by NodeRestrictor and PathRestrictor implementations.
+The TorCtl package is now fully pydoced. From this directory, you can
+do:
+# pydoc TorCtl.TorCtl
+and
+# pydoc TorCtl.PathSupport
+
+
2. metatroller.py
Metatroller observes the paths created by PathSupport and gathers
@@ -34,7 +41,11 @@
failures and stream failures. It also provides a meta control port for
use by Tor scanners.
+The metatroller is also pydoced. To view the documentation, you can do:
+# pydoc metatroller
+
+
3. soat.pl
SoaT scans exit nodes to verify that SSL, SSH, and Web connections
Modified: torflow/trunk/TorCtl/PathSupport.py
===================================================================
--- torflow/trunk/TorCtl/PathSupport.py 2008-05-03 09:28:43 UTC (rev 14541)
+++ torflow/trunk/TorCtl/PathSupport.py 2008-05-03 09:30:18 UTC (rev 14542)
@@ -1,5 +1,44 @@
#!/usr/bin/python
+"""
+Support classes for path construction
+
+The PathSupport package builds on top of TorCtl.TorCtl. It provides a
+number of interfaces that make path construction easier.
+
+The inheritance diagram for event handling is as follows:
+TorCtl.EventHandler <- PathBuilder <- CircuitHandler <- StreamHandler.
+
+Basically, EventHandler is what gets all the control port events
+packaged in nice clean classes (see help(TorCtl) for information on
+those).
+
+PathBuilder inherits from EventHandler and is what builds all circuits
+based on the requirements specified in the SelectionManager instance
+passed to its constructor. It also handles attaching streams to
+circuits. It only handles one building one circuit at a time.
+
+CircuitHandler optionally inherits from PathBuilder, and overrides its
+circuit event handling to manage building a pool of circuits as opposed
+to just one. It still uses the SelectionManager for path selection.
+
+StreamHandler inherits from CircuitHandler, and is what governs the
+attachment of an incoming stream on to one of the multiple circuits of
+the circuit handler.
+
+The SelectionManager is essentially a configuration wrapper around the
+most elegant portions of TorFlow: NodeGenerators, NodeRestrictions, and
+PathRestrictions. In the SelectionManager, a NodeGenerator is used to
+choose the nodes probabilistically according to some distribution while
+obeying the NodeRestrictions. These generators (one per hop) are handed
+off to the PathSelector, which uses the generators to build a complete
+path that satisfies the PathRestriction requirements.
+
+Have a look at the class hierarchy directly below to get a feel for how
+the restrictions fit together, and what options are available.
+
+"""
+
import TorCtl
import re
import struct
@@ -13,58 +52,77 @@
__all__ = ["NodeRestrictionList", "PathRestrictionList",
"PercentileRestriction", "OSRestriction", "ConserveExitsRestriction",
"FlagsRestriction", "MinBWRestriction", "VersionIncludeRestriction",
-"VersionExcludeRestriction", "ExitPolicyRestriction", "OrNodeRestriction",
+"VersionExcludeRestriction", "ExitPolicyRestriction", "NodeRestriction",
+"PathRestriction", "OrNodeRestriction", "MetaNodeRestriction",
"AtLeastNNodeRestriction", "NotNodeRestriction", "Subnet16Restriction",
-"UniqueRestriction", "UniformGenerator", "OrderedExitGenerator",
-"BwWeightedGenerator", "PathSelector", "Connection", "NickRestriction",
-"IdHexRestriction", "PathBuilder", "CircuitHandler", "StreamHandler",
-"SelectionManager", "CountryCodeRestriction", "CountryRestriction",
-"UniqueCountryRestriction", "SingleCountryRestriction",
-"ContinentRestriction", "ContinentJumperRestriction",
+"UniqueRestriction", "NodeGenerator", "UniformGenerator",
+"OrderedExitGenerator", "BwWeightedGenerator", "PathSelector",
+"Connection", "NickRestriction", "IdHexRestriction", "PathBuilder",
+"CircuitHandler", "StreamHandler", "SelectionManager",
+"CountryCodeRestriction", "CountryRestriction",
+"UniqueCountryRestriction", "SingleCountryRestriction",
+"ContinentRestriction", "ContinentJumperRestriction",
"UniqueContinentRestriction"]
#################### Path Support Interfaces #####################
class NodeRestriction:
"Interface for node restriction policies"
- def r_is_ok(self, r): return True
+ def r_is_ok(self, r):
+ "Returns true if Router 'r' is acceptable for this restriction"
+ return True
class NodeRestrictionList:
+ "Class to manage a list of NodeRestrictions"
def __init__(self, restrictions):
+ "Constructor. 'restrictions' is a list of NodeRestriction instances"
self.restrictions = restrictions
def r_is_ok(self, r):
+ "Returns true of Router 'r' passes all of the contained restrictions"
for rs in self.restrictions:
if not rs.r_is_ok(r): return False
return True
def add_restriction(self, restr):
+ "Add a NodeRestriction 'restr' to the list of restrictions"
self.restrictions.append(restr)
# TODO: This does not collapse meta restrictions..
def del_restriction(self, RestrictionClass):
+ """Remove all restrictions of type RestrictionClass from the list.
+ Does NOT inspect or collapse MetaNode Restrictions (though
+ MetaRestrictions can be removed if RestrictionClass is
+ MetaNodeRestriction)"""
self.restrictions = filter(
lambda r: not isinstance(r, RestrictionClass),
self.restrictions)
class PathRestriction:
"Interface for path restriction policies"
- def path_is_ok(self, path): return True
+ def path_is_ok(self, path):
+ "Return true if the list of Routers in path satisfies this restriction"
+ return True
class PathRestrictionList:
+ """Class to manage a list of PathRestrictions"""
def __init__(self, restrictions):
+ "Constructor. 'restrictions' is a list of PathRestriction instances"
self.restrictions = restrictions
def path_is_ok(self, path):
+ "Given list if Routers in 'path', check it against each restriction."
for rs in self.restrictions:
if not rs.path_is_ok(path):
return False
return True
def add_restriction(self, rstr):
+ "Add a PathRestriction 'rstr' to the list"
self.restrictions.append(rstr)
def del_restriction(self, RestrictionClass):
+ "Remove all PathRestrictions of type RestrictionClass from the list."
self.restrictions = filter(
lambda r: not isinstance(r, RestrictionClass),
self.restrictions)
@@ -72,26 +130,37 @@
class NodeGenerator:
"Interface for node generation"
def __init__(self, sorted_r, rstr_list):
+ """Constructor. Takes a bandwidth-sorted list of Routers 'sorted_r'
+ and a NodeRestrictionList 'rstr_list'"""
self.rstr_list = rstr_list # Check me before you yield!
self.sorted_r = sorted_r
self.rewind()
def reset_restriction(self, rstr_list):
+ "Reset the restriction list to a new list"
self.rstr_list = rstr_list
def rewind(self):
+ "Rewind the generator to the 'beginning'"
self.routers = copy.copy(self.sorted_r)
def mark_chosen(self, r):
+ """Mark a router as chosen: remove it from the list of routers
+ that can be returned in the future"""
self.routers.remove(r)
def all_chosen(self):
+ "Return true if all the routers have been marked as chosen"
return not self.routers
- def next_r(self): raise NotImplemented()
+ def next_r(self):
+ "Yield the next router according to the policy"
+ raise NotImplemented()
class Connection(TorCtl.Connection):
+ """Extended Connection class that provides a method for building circuits"""
def build_circuit(self, pathlen, path_sel):
+ "Tell Tor to build a circuit chosen by the PathSelector 'path_sel'"
circ = Circuit()
circ.path = path_sel.build_path(pathlen)
circ.exit = circ.path[pathlen-1]
@@ -118,14 +187,21 @@
# Exit->destination hops
class PercentileRestriction(NodeRestriction):
+ """Restriction to cut out a percentile slice of the network."""
def __init__(self, pct_skip, pct_fast, r_list):
+ """Constructor. Sets up the restriction such that routers in the
+ 'pct_skip' to 'pct_fast' percentile of bandwidth rankings are
+ returned from the sorted list 'r_list'"""
self.pct_fast = pct_fast
self.pct_skip = pct_skip
self.sorted_r = r_list
def r_is_ok(self, r):
+ "Returns true if r is in the percentile boundaries (by rank)"
# Hrmm.. technically we shouldn't count non-running routers in this..
- # but that is tricky to do efficiently
+ # but that is tricky to do efficiently.
+ # XXX: Is there any reason why sorted_r should have non-running
+ # routers in the first place?
if r.list_rank < len(self.sorted_r)*self.pct_skip/100: return False
elif r.list_rank > len(self.sorted_r)*self.pct_fast/100: return False
@@ -133,11 +209,15 @@
return True
class OSRestriction(NodeRestriction):
+ "Restriction based on operating system"
def __init__(self, ok, bad=[]):
+ """Constructor. Accept router OSes that match regexes in 'ok',
+ rejects those that match regexes in 'bad'."""
self.ok = ok
self.bad = bad
def r_is_ok(self, r):
+ "Returns true if r is in 'ok', false if 'r' is in 'bad'. If 'ok'"
for y in self.ok:
if re.search(y, r.os):
return True
@@ -148,11 +228,15 @@
if self.bad: return True
class ConserveExitsRestriction(NodeRestriction):
- # FIXME: Make this adaptive
+ "Restriction to reject exits from selection"
+ # XXX: Make this adaptive by ip/port
def r_is_ok(self, r): return not "Exit" in r.flags
class FlagsRestriction(NodeRestriction):
+ "Restriction for mandatory and forbidden router flags"
def __init__(self, mandatory, forbidden=[]):
+ """Constructor. 'mandatory' and 'forbidden' are both lists of router
+ flags as strings."""
self.mandatory = mandatory
self.forbidden = forbidden
@@ -183,32 +267,42 @@
return router.idhex == self.idhex
class MinBWRestriction(NodeRestriction):
+ """Require a minimum bandwidth"""
def __init__(self, minbw):
self.min_bw = minbw
def r_is_ok(self, router): return router.bw >= self.min_bw
class VersionIncludeRestriction(NodeRestriction):
+ """Require that the version match one in the list"""
def __init__(self, eq):
+ "Constructor. 'eq' is a list of versions as strings"
self.eq = map(TorCtl.RouterVersion, eq)
def r_is_ok(self, router):
+ """Returns true if the version of 'router' matches one of the
+ specified versions."""
for e in self.eq:
if e == router.version:
return True
return False
class VersionExcludeRestriction(NodeRestriction):
+ """Require that the version not match one in the list"""
def __init__(self, exclude):
+ "Constructor. 'exclude' is a list of versions as strings"
self.exclude = map(TorCtl.RouterVersion, exclude)
def r_is_ok(self, router):
+ """Returns false if the version of 'router' matches one of the
+ specified versions."""
for e in self.exclude:
if e == router.version:
return False
return True
class VersionRangeRestriction(NodeRestriction):
+ """Require that the versions be inside a specified range"""
def __init__(self, gr_eq, less_eq=None):
self.gr_eq = TorCtl.RouterVersion(gr_eq)
if less_eq: self.less_eq = TorCtl.RouterVersion(less_eq)
@@ -219,6 +313,7 @@
(not self.less_eq or router.version <= self.less_eq)
class ExitPolicyRestriction(NodeRestriction):
+ """Require that a router exit to an ip+port"""
def __init__(self, to_ip, to_port):
self.to_ip = to_ip
self.to_port = to_port
@@ -226,28 +321,37 @@
def r_is_ok(self, r): return r.will_exit_to(self.to_ip, self.to_port)
class MetaNodeRestriction(NodeRestriction):
+ """Interface for a NodeRestriction that is an expression consisting of
+ multiple other NodeRestrictions"""
# TODO: these should collapse the restriction and return a new
# instance for re-insertion (or None)
def next_rstr(self): raise NotImplemented()
def del_restriction(self, RestrictionClass): raise NotImplemented()
class OrNodeRestriction(MetaNodeRestriction):
+ """MetaNodeRestriction that is the boolean or of two or more
+ NodeRestrictions"""
def __init__(self, rs):
+ "Constructor. 'rs' is a list of NodeRestrictions"
self.rstrs = rs
def r_is_ok(self, r):
+ "Returns true if one of 'rs' is true for this router"
for rs in self.rstrs:
if rs.r_is_ok(r):
return True
return False
class NotNodeRestriction(MetaNodeRestriction):
+ """Negates a single restriction"""
def __init__(self, a):
self.a = a
def r_is_ok(self, r): return not self.a.r_is_ok(r)
class AtLeastNNodeRestriction(MetaNodeRestriction):
+ """MetaNodeRestriction that is true if at least n member
+ restrictions are true."""
def __init__(self, rstrs, n):
self.rstrs = rstrs
self.n = n
@@ -264,6 +368,8 @@
#################### Path Restrictions #####################
class Subnet16Restriction(PathRestriction):
+ """PathRestriction that mandates that no two nodes from the same
+ /16 subnet be in the path"""
def path_is_ok(self, path):
mask16 = struct.unpack(">I", socket.inet_aton("255.255.0.0"))[0]
ip16 = path[0].ip & mask16
@@ -273,6 +379,8 @@
return True
class UniqueRestriction(PathRestriction):
+ """Path restriction that mandates that the same router can't appear more
+ than once in a path"""
def path_is_ok(self, path):
for i in xrange(0,len(path)):
if path[i] in path[:i]:
@@ -287,7 +395,7 @@
return r.country_code != None
class CountryRestriction(NodeRestriction):
- """ Ensure a specific country_code for nodes """
+ """ Only accept nodes that are in 'country_code' """
def __init__(self, country_code):
self.country_code = country_code
@@ -382,12 +490,15 @@
#################### Node Generators ######################
class UniformGenerator(NodeGenerator):
+ """NodeGenerator that produces nodes in the uniform distribution"""
def next_r(self):
while not self.all_chosen():
r = random.choice(self.routers)
if self.rstr_list.r_is_ok(r): yield r
class OrderedExitGenerator(NodeGenerator):
+ """NodeGenerator that produces exits in an ordered fashion for a
+ specific port"""
def __init__(self, to_port, sorted_r, rstr_list):
self.to_port = to_port
self.next_exit_by_port = {}
@@ -421,8 +532,8 @@
break
class BwWeightedGenerator(NodeGenerator):
- """ Pass exit=True to create a generator for exit-nodes """
def __init__(self, sorted_r, rstr_list, pathlen, exit=False):
+ """ Pass exit=True to create a generator for exit-nodes """
# Out for an exit-node?
self.exit = exit
# Different sums of bandwidths
@@ -505,14 +616,21 @@
pass
class PathSelector:
- "Implementation of path selection policies"
+ """Implementation of path selection policies. Builds a path according
+ to entry, middle, and exit generators that satisfies the path
+ restrictions."""
def __init__(self, entry_gen, mid_gen, exit_gen, path_restrict):
+ """Constructor. The first three arguments are NodeGenerators with
+ their appropriate restrictions. The 'path_restrict' is a
+ PathRestrictionList"""
self.entry_gen = entry_gen
self.mid_gen = mid_gen
self.exit_gen = exit_gen
self.path_restrict = path_restrict
def build_path(self, pathlen):
+ """Creates a path of 'pathlen' hops, and returns it as a list of
+ Router instances"""
self.entry_gen.rewind()
self.mid_gen.rewind()
self.exit_gen.rewind()
@@ -550,11 +668,10 @@
"""Helper class to handle configuration updates
The methods are NOT threadsafe. They may ONLY be called from
- EventHandler's thread.
-
- To update the selection manager, schedule a config update job
- using PathBuilder.schedule_selmgr() with a worker function
- to modify this object.
+ EventHandler's thread. This means that to update the selection
+ manager, you must schedule a config update job using
+ PathBuilder.schedule_selmgr() with a worker function to modify
+ this object.
"""
def __init__(self, pathlen, order_exits,
percent_fast, percent_skip, min_bw, use_all_exits,
@@ -572,6 +689,8 @@
self.geoip_config = geoip_config
def reconfigure(self, sorted_r):
+ """This function is called after a configuration change,
+ to rebuild the RestrictionLists."""
if self.use_all_exits:
self.path_rstr = PathRestrictionList([UniqueRestriction()])
else:
@@ -679,6 +798,7 @@
return
def set_target(self, ip, port):
+ "Called to update the ExitPolicyRestrictions with a new ip and port"
self.exit_rstr.del_restriction(ExitPolicyRestriction)
self.exit_rstr.add_restriction(ExitPolicyRestriction(ip, port))
if self.__ordered_exit_gen: self.__ordered_exit_gen.set_port(port)
@@ -699,6 +819,7 @@
self.exit_rstr.add_restriction(CountryRestriction(self.geoip_config.exit_country))
class Circuit:
+ "Class to describe a circuit"
def __init__(self):
self.circ_id = 0
self.path = [] # routers
@@ -712,9 +833,12 @@
self.setup_duration = None # Sum of extend-times
self.pending_streams = [] # Which stream IDs are pending us
- def id_path(self): return map(lambda r: r.idhex, self.path)
+ def id_path(self):
+ "Returns a list of idhex keys for the path of Routers"
+ return map(lambda r: r.idhex, self.path)
class Stream:
+ "Class to describe a stream"
def __init__(self, sid, host, port, kind):
self.strm_id = sid
self.detached_from = [] # circ id #'s
@@ -729,7 +853,9 @@
self.failed = False
self.failed_reason = None # Cheating a little.. Only used by StatsHandler
- def lifespan(self, now): return now-self.attached_at
+ def lifespan(self, now):
+ "Returns the age of the stream"
+ return now-self.attached_at
# TODO: Make passive "PathWatcher" so people can get aggregate
# node reliability stats for normal usage without us attaching streams
@@ -744,6 +870,9 @@
of the EventHandler.
"""
def __init__(self, c, selmgr, RouterClass):
+ """Constructor. 'c' is a Connection, 'selmgr' is a SelectionManager,
+ and 'RouterClass' is a class that inherits from Router and is used
+ to create annotated Routers."""
TorCtl.EventHandler.__init__(self)
self.c = c
nslist = c.get_network_status()
@@ -792,6 +921,9 @@
def heartbeat_event(self, event):
+ """This function handles dispatching scheduled jobs. If you
+ extend PathBuilder and want to implement this function for
+ some reason, be sure to call the parent class"""
while not self.imm_jobs.empty():
imm_job = self.imm_jobs.get_nowait()
imm_job(self)
@@ -848,6 +980,7 @@
return self.selmgr.path_selector.build_path(self.selmgr.pathlen)
def attach_stream_any(self, stream, badcircs):
+ "Attach a stream to a valid circuit, avoiding any in 'badcircs'"
# Newnym, and warn if not built plus pending
unattached_streams = [stream]
if self.new_nym:
@@ -1034,8 +1167,13 @@
################### CircuitHandler #############################
class CircuitHandler(PathBuilder):
- """ CircuitHandler that extends from PathBuilder """
+ """ CircuitHandler that extends from PathBuilder to handle multiple
+ circuits as opposed to just one. """
def __init__(self, c, selmgr, num_circuits, RouterClass):
+ """Constructor. 'c' is a Connection, 'selmgr' is a SelectionManager,
+ 'num_circuits' is the number of circuits to keep in the pool,
+ and 'RouterClass' is a class that inherits from Router and is used
+ to create annotated Routers."""
PathBuilder.__init__(self, c, selmgr, RouterClass)
# Set handler to the connection here to
# not miss any circuit events on startup
@@ -1143,7 +1281,9 @@
################### StreamHandler ##############################
class StreamHandler(CircuitHandler):
- """ StreamHandler that extends from the CircuitHandler """
+ """ StreamHandler that extends from the CircuitHandler
+ to handle attaching streams to an appropriate circuit
+ in the pool. """
def __init__(self, c, selmgr, num_circs, RouterClass):
CircuitHandler.__init__(self, c, selmgr, num_circs, RouterClass)
self.sorted_circs = None # optional sorted list
Modified: torflow/trunk/TorCtl/README
===================================================================
--- torflow/trunk/TorCtl/README 2008-05-03 09:28:43 UTC (rev 14541)
+++ torflow/trunk/TorCtl/README 2008-05-03 09:30:18 UTC (rev 14542)
@@ -1,5 +1,10 @@
-TODO: Write me.
+See the pydoc:
-In the mean time, try
-svn co https://tor-svn.freehaven.net/svn/torctl/trunk/doc/howto.txt
+# python
+>>> import TorCtl
+>>> help(TorCtl)
+
+>>> import PathSupport
+>>> help(PathSupport)
+
Modified: torflow/trunk/TorCtl/TorCtl.py
===================================================================
--- torflow/trunk/TorCtl/TorCtl.py 2008-05-03 09:28:43 UTC (rev 14541)
+++ torflow/trunk/TorCtl/TorCtl.py 2008-05-03 09:30:18 UTC (rev 14542)
@@ -4,10 +4,32 @@
# Copyright 2007 Mike Perry. See LICENSE file.
"""
-TorCtl -- Library to control Tor processes.
+Library to control Tor processes.
+
+This library handles sending commands, parsing responses, and delivering
+events to and from the control port. The basic usage is to create a
+socket, wrap that in a TorCtl.Connection, and then add an EventHandler
+to that connection. A simple example with a DebugEventHandler (that just
+echoes the events back to stdout) is present in run_example().
+
+Note that the TorCtl.Connection is fully compatible with the more
+advanced EventHandlers in TorCtl.PathSupport (and of course any other
+custom event handlers that you may extend off of those).
+
+This package also contains a helper class for representing Routers, and
+classes and constants for each event.
+
"""
-# XXX: Docstring all exported classes/interfaces. Also need __all__
+__all__ = ["EVENTTYPE", "CIRC", "STREAM", "ORCONN", "STREAM_BW", "BW",
+ "NS", "NEWDESC", "ADDRMAP", "DEBUG", "INFO", "NOTICE", "WARN",
+ "ERR", "TorCtlError", "TorCtlClosed", "ProtocolError",
+ "ErrorReply", "NetworkStatus", "ExitPolicyLine", "Router",
+ "RouterVersion", "Connection", "parse_ns_body",
+ "EventHandler", "DebugEventHandler", "NetworkStatusEvent",
+ "NewDescEvent", "CircuitEvent", "StreamEvent", "ORConnEvent",
+ "StreamBwEvent", "LogEvent", "AddrMapEvent", "BWEvent",
+ "UnknownEvent" ]
import os
import re
@@ -151,6 +173,8 @@
self.event_string = event_string
class ExitPolicyLine:
+ """ Class to represent a line in a Router's exit policy in a way
+ that can be easily checked. """
def __init__(self, match, ip_mask, port_low, port_high):
self.match = match
if ip_mask == "*":
@@ -177,6 +201,8 @@
self.port_high = int(port_high)
def check(self, ip, port):
+ """Check to see if an ip and port is matched by this line.
+ Returns true if the line is an Accept, and False if it is a Reject. """
ip = struct.unpack(">I", socket.inet_aton(ip))[0]
if (ip & self.netmask) == self.ip:
if self.port_low <= port and port <= self.port_high:
@@ -184,6 +210,8 @@
return -1
class RouterVersion:
+ """ Represents a Router's version. Overloads all comparison operators
+ to check for newer, older, or equivalent versions. """
def __init__(self, version):
if version:
v = re.search("^(\d+).(\d+).(\d+).(\d+)", version).groups()
@@ -202,6 +230,11 @@
def __str__(self): return self.ver_string
class Router:
+ """
+ Class to represent a router from a descriptor. Can either be
+ created from the parsed fields, or can be built from a
+ descriptor+NetworkStatus
+ """
def __init__(self, idhex, name, bw, down, exitpolicy, flags, ip, version, os, uptime):
self.idhex = idhex
self.nickname = name
@@ -216,6 +249,13 @@
self.uptime = uptime
def build_from_desc(desc, ns):
+ """
+ Static method of Router that parses a descriptor string into this class.
+ 'desc' is a full descriptor as a string.
+ 'ns' is a TorCtl.NetworkStatus instance for this router (needed for
+ the flags, the nickname, and the idhex string).
+ Returns a Router instance.
+ """
# XXX: Compile these regular expressions? This is an expensive process
# Use http://docs.python.org/lib/profile.html to verify this is
# the part of startup that is slow
@@ -264,6 +304,8 @@
build_from_desc = Callable(build_from_desc)
def update_to(self, new):
+ """ Somewhat hackish method to update this router to be a copy of
+ 'new' """
if self.idhex != new.idhex:
plog("ERROR", "Update of router "+self.nickname+"changes idhex!")
self.idhex = new.idhex
@@ -277,6 +319,8 @@
self.uptime = new.uptime
def will_exit_to(self, ip, port):
+ """ Check the entire exitpolicy to see if the router will allow
+ connections to 'ip':'port' """
for line in self.exitpolicy:
ret = line.check(ip, port)
if ret != -1:
@@ -285,7 +329,8 @@
return False
class Connection:
- """A Connection represents a connection to the Tor process."""
+ """A Connection represents a connection to the Tor process via the
+ control port."""
def __init__(self, sock):
"""Create a Connection to communicate with the Tor process over the
socket 'sock'.
@@ -454,7 +499,7 @@
"""Cause future events from the Tor process to be sent to 'handler'.
"""
self._handler = handler
- self._handleFn = handler.handle1
+ self._handleFn = handler._handle1
def _read_reply(self):
lines = []
@@ -572,7 +617,8 @@
self.sendAndRecv("RESETCONF %s\r\n"%(" ".join(keylist)))
def get_network_status(self, who="all"):
- """Get the entire network status list"""
+ """Get the entire network status list. Returns a list of
+ TorCtl.NetworkStatus instances."""
return parse_ns_body(self.sendAndRecv("GETINFO ns/"+who+"\r\n")[0][2])
def get_router(self, ns):
@@ -582,6 +628,9 @@
def read_routers(self, nslist):
+ """ Given a list a NetworkStatuses in 'nslist', this function will
+ return a list of new Router instances.
+ """
bad_key = 0
new = []
for ns in nslist:
@@ -656,6 +705,7 @@
self.sendAndRecv("RESOLVE %s\r\n"%host)
def map_address(self, kvList):
+ """ Sends the MAPADDRESS command for each of the tuples in kvList """
if not kvList:
return
m = " ".join([ "%s=%s" for k,v in kvList])
@@ -693,9 +743,9 @@
self.sendAndRecv("REDIRECTSTREAM %d %s\r\n"%(streamid, newaddr))
def attach_stream(self, streamid, circid, hop=None):
- """Attach a stream to a circuit, specify both by IDs.
- If hop is given, try to use the specified hop in
- the circuit as the exit node for this stream.
+ """Attach a stream to a circuit, specify both by IDs. If hop is given,
+ try to use the specified hop in the circuit as the exit node for
+ this stream.
"""
if hop:
self.sendAndRecv("ATTACHSTREAM %d %d HOP=%d\r\n"%(streamid, circid, hop))
@@ -718,7 +768,8 @@
self.sendAndRecv("+POSTDESCRIPTOR\r\n%s"%escape_dots(desc))
def parse_ns_body(data):
- "Parse the body of an NS event or command."
+ """Parse the body of an NS event or command into a list of
+ NetworkStatus instances"""
nsgroups = re.compile(r"^r ", re.M).split(data)
nsgroups.pop(0)
nslist = []
@@ -731,7 +782,9 @@
return nslist
class EventHandler:
- """An 'EventHandler' wraps callbacks for the events Tor can return."""
+ """An 'EventHandler' wraps callbacks for the events Tor can return.
+ Each event argument is an instance of the corresponding event
+ class."""
def __init__(self):
"""Create a new EventHandler."""
self._map1 = {
@@ -750,15 +803,15 @@
"NS" : self.ns_event
}
- def handle1(self, timestamp, lines):
+ def _handle1(self, timestamp, lines):
"""Dispatcher: called from Connection when an event is received."""
for code, msg, data in lines:
- event = self.decode1(msg, data)
+ event = self._decode1(msg, data)
event.arrived_at = timestamp
self.heartbeat_event(event)
self._map1.get(event.event_name, self.unknown_event)(event)
- def decode1(self, body, data):
+ def _decode1(self, body, data):
"""Unpack an event message into a type/arguments-tuple tuple."""
if " " in body:
evtype,body = body.split(" ",1)
@@ -977,6 +1030,9 @@
return host, port
def run_example(host,port):
+ """ Example of basic TorCtl usage. See PathSupport for more advanced
+ usage.
+ """
print "host is %s:%d"%(host,port)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host,port))
Modified: torflow/trunk/TorCtl/TorUtil.py
===================================================================
--- torflow/trunk/TorCtl/TorUtil.py 2008-05-03 09:28:43 UTC (rev 14541)
+++ torflow/trunk/TorCtl/TorUtil.py 2008-05-03 09:30:18 UTC (rev 14542)
@@ -17,10 +17,10 @@
__all__ = ["Enum", "Enum2", "Callable", "sort_list", "quote", "escape_dots", "unescape_dots",
"BufSock", "secret_to_key", "urandom_rng", "s2k_gen", "s2k_check", "plog",
- "ListenSocket", "zprob"]
+ "ListenSocket", "zprob"]
class Enum:
- # Helper: define an ordered dense name-to-number 1-1 mapping.
+ """ Defines an ordered dense name-to-number 1-1 mapping """
def __init__(self, start, names):
self.nameOf = {}
idx = start
@@ -30,7 +30,7 @@
idx += 1
class Enum2:
- # Helper: define an ordered sparse name-to-number 1-1 mapping.
+ """ Defines an ordered sparse name-to-number 1-1 mapping """
def __init__(self, **args):
self.__dict__.update(args)
self.nameOf = {}
Modified: torflow/trunk/metatroller.py
===================================================================
--- torflow/trunk/metatroller.py 2008-05-03 09:28:43 UTC (rev 14541)
+++ torflow/trunk/metatroller.py 2008-05-03 09:30:18 UTC (rev 14542)
@@ -2,7 +2,14 @@
# Metatroller.
"""
-Metatroller - Tor Meta controller
+Tor Meta controller
+
+The Metatroller uses TorCtl.PathSupport to build a meta-controller that
+listens for commands on a local TCP port. In addition, it gathers a
+large amount of statistics on circuit failure rates, streams failure
+rates, stream bandwidths, probabilities of bandwidth ratios, and much
+much more.
+
"""
import atexit
@@ -44,7 +51,10 @@
use_exit=None,
use_guards=False)
+# FIXME: Much of this should be moved into TorCtl/StatsSupport.py
+
class BandwidthStats:
+ "Class that manages observed bandwidth through a Router"
def __init__(self):
self.byte_list = []
self.duration_list = []
@@ -54,6 +64,7 @@
self.dev = 0
def _exp(self): # Weighted avg
+ "Expectation - weighted average of the bandwidth through this node"
tot_bw = reduce(lambda x, y: x+y, self.byte_list, 0.0)
EX = 0.0
for i in xrange(len(self.byte_list)):
@@ -63,6 +74,7 @@
return EX
def _exp2(self): # E[X^2]
+ "Second moment of the bandwidth"
tot_bw = reduce(lambda x, y: x+y, self.byte_list, 0.0)
EX = 0.0
for i in xrange(len(self.byte_list)):
@@ -72,6 +84,7 @@
return EX
def _dev(self): # Weighted dev
+ "Standard deviation of bandwidth"
EX = self.mean
EX2 = self._exp2()
arg = EX2 - (EX*EX)
@@ -80,6 +93,7 @@
return math.sqrt(abs(arg))
def add_bw(self, bytes, duration):
+ "Add an observed transfer of 'bytes' for 'duration' seconds"
if not bytes: plog("WARN", "No bytes for bandwidth")
bytes /= 1024.
self.byte_list.append(bytes)
@@ -94,11 +108,15 @@
# Technically we could just add member vars as we need them, but this
# is a bit more clear
class StatsRouter(TorCtl.Router):
+ "Extended Router to handle statistics markup"
def __init__(self, router): # Promotion constructor :)
+ """'Promotion Constructor' that converts a Router directly into a
+ StatsRouter without a copy."""
self.__dict__ = router.__dict__
self.reset()
def reset(self):
+ "Reset all stats on this Router"
self.circ_uncounted = 0
self.circ_failed = 0
self.circ_succeeded = 0 # disjoint from failed
@@ -129,11 +147,15 @@
self.prob_zb = 0
def avg_extend_time(self):
+ """Return the average amount of time it took for this router
+ to extend a circuit one hop"""
if self.total_extended:
return self.total_extend_time/self.total_extended
else: return 0
def bw_ratio(self):
+ """Return the ratio of the Router's advertised bandwidth to its
+ observed average stream bandwidth"""
bw = self.bwstats.mean
if bw == 0.0: return 0
else: return self.bw/(1024.*bw)
@@ -147,10 +169,14 @@
else: return ret
def failed_per_hour(self):
+ """Return the number of circuit extend failures per hour for this
+ Router"""
return (3600.*(self.circ_failed+self.strm_failed))/self.current_uptime()
# XXX: Seperate suspected from failed in totals
def suspected_per_hour(self):
+ """Return the number of circuits that failed with this router as an
+ earlier hop"""
return (3600.*(self.circ_suspected+self.strm_suspected
+self.circ_failed+self.strm_failed))/self.current_uptime()
@@ -196,6 +222,7 @@
+" U="+str(round(self.current_uptime()/3600, 1))+"\n")
def sanity_check(self):
+ "Makes sure all stats are self-consistent"
if (self.circ_failed + self.circ_succeeded + self.circ_suspected
+ self.circ_uncounted != self.circ_chosen):
plog("ERROR", self.nickname+" does not add up for circs")
@@ -228,7 +255,7 @@
+str(per_hour_tot) +" vs "+str(chosen_tot))
class ReasonRouterList:
- "Helper class to track which reasons are in which routers."
+ "Helper class to track which Routers have failed for a given reason"
def __init__(self, reason):
self.reason = reason
self.rlist = {}
@@ -236,6 +263,7 @@
def sort_list(self): raise NotImplemented()
def write_list(self, f):
+ "Write the list of failure counts for this reason 'f'"
rlist = self.sort_list()
for r in rlist:
susp = 0
@@ -251,9 +279,11 @@
f.write(str(susp)+"/"+str(tot_susp)+"\n")
def add_r(self, r):
+ "Add a router to the list for this reason"
self.rlist[r] = 1
def total_suspected(self):
+ "Get a list of total suspected failures for this reason"
# suspected is disjoint from failed. The failed table
# may not have an entry
def notlambda(x, y):
@@ -270,6 +300,7 @@
return reduce(notlambda, self.rlist.iterkeys(), 0)
def total_failed(self):
+ "Get a list of total failures for this reason"
def notlambda(x, y):
if self.reason in y.reason_failed:
return (x + y.reason_failed[self.reason])
@@ -277,6 +308,9 @@
return reduce(notlambda, self.rlist.iterkeys(), 0)
class SuspectRouterList(ReasonRouterList):
+ """Helper class to track all routers suspected of failing for a given
+ reason. The main difference between this and the normal
+ ReasonRouterList is the sort order and the verification."""
def __init__(self, reason): ReasonRouterList.__init__(self,reason)
def sort_list(self):
@@ -290,6 +324,9 @@
self.rlist.iterkeys(), 0)
class FailedRouterList(ReasonRouterList):
+ """Helper class to track all routers that failed for a given
+ reason. The main difference between this and the normal
+ ReasonRouterList is the sort order and the verification."""
def __init__(self, reason): ReasonRouterList.__init__(self,reason)
def sort_list(self):
@@ -304,6 +341,8 @@
class StatsHandler(PathSupport.PathBuilder):
+ """An extension of PathSupport.PathBuilder that keeps track of
+ router statistics for every circuit and stream"""
def __init__(self, c, slmgr):
PathBuilder.__init__(self, c, slmgr, StatsRouter)
self.circ_count = 0
@@ -314,6 +353,8 @@
self.suspect_reasons = {}
def run_zbtest(self): # Unweighted z-test
+ """Run unweighted z-test to calculate the probabilities of a node
+ having a given stream bandwidth based on the Normal distribution"""
n = reduce(lambda x, y: x+(y.bwstats.mean > 0), self.sorted_r, 0)
if n == 0: return (0, 0)
avg = reduce(lambda x, y: x+y.bwstats.mean, self.sorted_r, 0)/float(n)
@@ -329,6 +370,9 @@
return (avg, stddev)
def run_zrtest(self): # Unweighted z-test
+ """Run unweighted z-test to calculate the probabilities of a node
+ having a given ratio of stream bandwidth to advertised bandwidth
+ based on the Normal distribution"""
n = reduce(lambda x, y: x+(y.bw_ratio() > 0), self.sorted_r, 0)
if n == 0: return (0, 0)
avg = reduce(lambda x, y: x+y.bw_ratio(), self.sorted_r, 0)/float(n)
@@ -344,6 +388,7 @@
return (avg, stddev)
def write_reasons(self, f, reasons, name):
+ "Write out all the failure reasons and statistics for all Routers"
f.write("\n\n\t----------------- "+name+" -----------------\n")
for rsn in reasons:
f.write("\n"+rsn.reason+". Failed: "+str(rsn.total_failed())
@@ -351,12 +396,14 @@
rsn.write_list(f)
def write_routers(self, f, rlist, name):
+ "Write out all the usage statistics for all Routers"
f.write("\n\n\t----------------- "+name+" -----------------\n\n")
for r in rlist:
# only print it if we've used it.
if r.circ_chosen+r.strm_chosen > 0: f.write(str(r))
def write_stats(self, filename):
+ "Write out all the statistics the StatsHandler has gathered"
# TODO: all this shit should be configurable. Some of it only makes
# sense when scanning in certain modes.
plog("DEBUG", "Writing stats")
@@ -520,6 +567,7 @@
PathBuilder.circ_status_event(self, c)
def count_stream_reason_failed(self, s, reason):
+ "Count the routers involved in a failure"
# Update failed count,reason_failed for exit
r = self.circuits[s.circ_id].exit
if not reason in r.reason_failed: r.reason_failed[reason] = 1
@@ -530,6 +578,7 @@
self.failed_reasons[reason].add_r(r)
def count_stream_suspects(self, s, lreason, reason):
+ "Count the routers 'suspected' of being involved in a failure"
if lreason in ("TIMEOUT", "INTERNAL", "TORPROTOCOL" "DESTROY"):
for r in self.circuits[s.circ_id].path[:-1]:
r.strm_suspected += 1
@@ -635,6 +684,7 @@
plog("DEBUG", msg)
def commandloop(s, c, h):
+ "The main metatroller listener loop"
s.write("220 Welcome to the Tor Metatroller "+mt_version+"! Try HELP for Info\r\n\r\n")
while 1:
buf = s.readline()