[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [chutney/master] Implement chutney performance testing
commit 51dc1a3c2b3d2e095af077a8987ef1c7b469ea89
Author: teor <teor2345@xxxxxxxxx>
Date: Mon Jul 6 16:24:56 2015 +1000
Implement chutney performance testing
The following environmental variables affect chutney verify:
CHUTNEY_DATA_BYTES=n sends n bytes per test connection (10 KBytes)
CHUTNEY_CONNECTIONS=n makes n test connections per client (1)
CHUTNEY_HS_MULTI_CLIENT=1 makes each client connect to each HS (0)
When enough data is transmitted, chutney verify reports:
Single Stream Bandwidth: the speed of the slowest stream, end-to-end
Overall tor Bandwidth: the sum of the bandwidth across each tor instance
This approximates the CPU-bound tor performance on the current machine,
assuming everything is multithreaded and network performance is infinite.
These new features are all documented in the README.
---
README | 39 ++++++++-
lib/chutney/TorNet.py | 223 +++++++++++++++++++++++++++++++++++++++++-------
lib/chutney/Traffic.py | 97 ++++++++++++++++++---
3 files changed, 316 insertions(+), 43 deletions(-)
diff --git a/README b/README
index a8b5660..3e4b1f5 100644
--- a/README
+++ b/README
@@ -7,7 +7,7 @@ It is supposed to be a good tool for:
- Launching and monitoring a testing tor network
- Running tests on a testing tor network
-Right now it only sorta does the first two.
+Right now it only sorta does these things.
You will need, at the moment:
- Tor installed somewhere in your path or the location of the 'tor' and
@@ -16,12 +16,49 @@ You will need, at the moment:
- Python 2.7 or later
Stuff to try:
+
+Standard Actions:
./chutney configure networks/basic
./chutney start networks/basic
./chutney status networks/basic
+ ./chutney verify networks/basic
./chutney hup networks/basic
./chutney stop networks/basic
+Bandwidth Tests:
+ ./chutney configure networks/basic-min
+ ./chutney start networks/basic-min
+ ./chutney status networks/basic-min
+ CHUTNEY_DATA_BYTES=104857600 ./chutney verify networks/basic-min
+ # Send 100MB of data per client connection
+ # verify produces performance figures for:
+ # Single Stream Bandwidth: the speed of the slowest stream, end-to-end
+ # Overall tor Bandwidth: the sum of the bandwidth across each tor instance
+ # This approximates the CPU-bound tor performance on the current machine,
+ # assuming everything is multithreaded and network performance is infinite.
+ ./chutney stop networks/basic-min
+
+Connection Tests:
+ ./chutney configure networks/basic-025
+ ./chutney start networks/basic-025
+ ./chutney status networks/basic-025
+ CHUTNEY_CONNECTIONS=5 ./chutney verify networks/basic-025
+ # Make 5 connections from each client through a random exit
+ ./chutney stop networks/basic-025
+
+Note: If you create 7 or more connections to a hidden service from a single
+client, you'll likely get a verification failure due to
+https://trac.torproject.org/projects/tor/ticket/15937
+
+HS Connection Tests:
+ ./chutney configure networks/hs-025
+ ./chutney start networks/hs-025
+ ./chutney status networks/hs-025
+ CHUTNEY_HS_MULTI_CLIENT=1 ./chutney verify networks/hs-025
+ # Make a connection from each client to each hs
+ # Default behavior is one client connects to each HS
+ ./chutney stop networks/hs-025
+
The configuration files:
networks/basic holds the configuration for the network you're configuring
above. It refers to some torrc template files in torrc_templates/.
diff --git a/lib/chutney/TorNet.py b/lib/chutney/TorNet.py
index 439aa10..c79f64b 100644
--- a/lib/chutney/TorNet.py
+++ b/lib/chutney/TorNet.py
@@ -690,6 +690,14 @@ DEFAULTS = {
# Used when poll_launch_time is None, but RunAsDaemon is not set
# Set low so that we don't interfere with the voting interval
'poll_launch_time_default': 0.1,
+ # the number of bytes of random data we send on each connection
+ 'data_bytes': int(os.environ.get('CHUTNEY_DATA_BYTES', 10 * 1024)),
+ # the number of times each client will connect
+ 'connection_count': int(os.environ.get('CHUTNEY_CONNECTIONS', 1)),
+ # Do we want every client to connect to every HS, or one client
+ # to connect to each HS?
+ # (Clients choose an exit at random, so this doesn't apply to exits.)
+ 'hs_multi_client': int(os.environ.get('CHUTNEY_HS_MULTI_CLIENT', 0)),
}
@@ -908,46 +916,201 @@ class Network(object):
# HSs must have a HiddenServiceDir with
# "HiddenServicePort <HS_PORT> 127.0.0.1:<LISTEN_PORT>"
HS_PORT = 5858
- DATALEN = 10 * 1024 # Octets.
- TIMEOUT = 3 # Seconds.
- with open('/dev/urandom', 'r') as randfp:
- tmpdata = randfp.read(DATALEN)
+ # The amount of data to send between each source-sink pair,
+ # each time the source connects.
+ # We create a source-sink pair for each (bridge) client to an exit,
+ # and a source-sink pair for a (bridge) client to each hidden service
+ DATALEN = self._dfltEnv['data_bytes']
+ # Print a dot each time a sink verifies this much data
+ DOTDATALEN = 5 * 1024 * 1024 # Octets.
+ TIMEOUT = 3 # Seconds.
+ # Calculate the amount of random data we should use
+ randomlen = self._calculate_randomlen(DATALEN)
+ reps = self._calculate_reps(DATALEN, randomlen)
+ # sanity check
+ if reps == 0:
+ DATALEN = 0
+ # Get the random data
+ if randomlen > 0:
+ # print a dot after every DOTDATALEN data is verified, rounding up
+ dot_reps = self._calculate_reps(DOTDATALEN, randomlen)
+ # make sure we get at least one dot per transmission
+ dot_reps = min(reps, dot_reps)
+ with open('/dev/urandom', 'r') as randfp:
+ tmpdata = randfp.read(randomlen)
+ else:
+ dot_reps = 0
+ tmpdata = {}
+ # now make the connections
bind_to = ('127.0.0.1', LISTEN_PORT)
- tt = chutney.Traffic.TrafficTester(bind_to, tmpdata, TIMEOUT)
+ tt = chutney.Traffic.TrafficTester(bind_to,
+ tmpdata,
+ TIMEOUT,
+ reps,
+ dot_reps)
client_list = filter(lambda n:
n._env['tag'] == 'c' or n._env['tag'] == 'bc',
self._nodes)
+ exit_list = filter(lambda n:
+ ('exit' in n._env.keys()) and n._env['exit'] == 1,
+ self._nodes)
+ hs_list = filter(lambda n:
+ n._env['tag'] == 'h',
+ self._nodes)
if len(client_list) == 0:
print(" Unable to verify network: no client nodes available")
return False
- # Each client binds directly to 127.0.0.1:LISTEN_PORT via an Exit relay
- for op in client_list:
- print(" Exit to %s:%d via client %s:%s"
- % ('127.0.0.1', LISTEN_PORT,
- 'localhost', op._env['socksport']))
- tt.add(chutney.Traffic.Source(tt, bind_to, tmpdata,
- ('localhost',
- int(op._env['socksport']))))
- # The HS redirects .onion connections made to hs_hostname:HS_PORT
- # to the Traffic Tester's 127.0.0.1:LISTEN_PORT
- # We must have at least one working client for the hs test to succeed
- for hs in filter(lambda n:
- n._env['tag'] == 'h',
- self._nodes):
- # Instead of binding directly to LISTEN_PORT via an Exit relay,
- # we bind to hs_hostname:HS_PORT via a hidden service connection
- # through the first available client
- bind_to = (hs._env['hs_hostname'], HS_PORT)
- # Just choose the first client
- client = client_list[0]
- print(" HS to %s:%d (%s:%d) via client %s:%s"
- % (hs._env['hs_hostname'], HS_PORT,
+ if len(exit_list) == 0 and len(hs_list) == 0:
+ print(" Unable to verify network: no exit/hs nodes available")
+ print(" Exit nodes must be declared 'relay=1, exit=1'")
+ print(" HS nodes must be declared 'tag=\"hs\"'")
+ return False
+ print("Connecting:")
+ # the number of tor nodes in paths which will send DATALEN data
+ # if a node is used in two paths, we count it twice
+ # this is a lower bound, as cannabilised circuits are one node longer
+ total_path_node_count = 0
+ total_path_node_count += self._configure_exits(tt, bind_to,
+ tmpdata, reps,
+ client_list, exit_list,
+ LISTEN_PORT)
+ total_path_node_count += self._configure_hs(tt,
+ tmpdata, reps,
+ client_list, hs_list,
+ HS_PORT,
+ LISTEN_PORT)
+ print("Transmitting Data:")
+ start_time = time.clock()
+ status = tt.run()
+ end_time = time.clock()
+ # if we fail, don't report the bandwidth
+ if not status:
+ return status
+ # otherwise, report bandwidth used, if sufficient data was transmitted
+ self._report_bandwidth(DATALEN, total_path_node_count,
+ start_time, end_time)
+ return status
+
+ # In order to performance test a tor network, we need to transmit
+ # several hundred megabytes of data or more. Passing around this
+ # much data in Python has its own performance impacts, so we provide
+ # a smaller amount of random data instead, and repeat it to DATALEN
+ def _calculate_randomlen(self, datalen):
+ MAX_RANDOMLEN = 128 * 1024 # Octets.
+ if datalen > MAX_RANDOMLEN:
+ return MAX_RANDOMLEN
+ else:
+ return datalen
+
+ def _calculate_reps(self, datalen, replen):
+ # sanity checks
+ if datalen == 0 or replen == 0:
+ return 0
+ # effectively rounds datalen up to the nearest replen
+ if replen < datalen:
+ return (datalen + replen - 1) / replen
+ else:
+ return 1
+
+ # if there are any exits, each client / bridge client transmits
+ # via 4 nodes (including the client) to an arbitrary exit
+ # Each client binds directly to 127.0.0.1:LISTEN_PORT via an Exit relay
+ def _configure_exits(self, tt, bind_to,
+ tmpdata, reps,
+ client_list, exit_list,
+ LISTEN_PORT):
+ CLIENT_EXIT_PATH_NODES = 4
+ connection_count = self._dfltEnv['connection_count']
+ exit_path_node_count = 0
+ if len(exit_list) > 0:
+ exit_path_node_count += (len(client_list)
+ * CLIENT_EXIT_PATH_NODES
+ * connection_count)
+ for op in client_list:
+ print(" Exit to %s:%d via client %s:%s"
+ % ('127.0.0.1', LISTEN_PORT,
+ 'localhost', op._env['socksport']))
+ for i in range(connection_count):
+ tt.add(chutney.Traffic.Source(tt,
+ bind_to,
+ tmpdata,
+ ('localhost',
+ int(op._env['socksport'])),
+ reps))
+ return exit_path_node_count
+
+ # The HS redirects .onion connections made to hs_hostname:HS_PORT
+ # to the Traffic Tester's 127.0.0.1:LISTEN_PORT
+ # an arbitrary client / bridge client transmits via 8 nodes
+ # (including the client and hs) to each hidden service
+ # Instead of binding directly to LISTEN_PORT via an Exit relay,
+ # we bind to hs_hostname:HS_PORT via a hidden service connection
+ def _configure_hs(self, tt,
+ tmpdata, reps,
+ client_list, hs_list,
+ HS_PORT,
+ LISTEN_PORT):
+ CLIENT_HS_PATH_NODES = 8
+ connection_count = self._dfltEnv['connection_count']
+ hs_path_node_count = (len(hs_list)
+ * CLIENT_HS_PATH_NODES
+ * connection_count)
+ # Each client in hs_client_list connects to each hs
+ if self._dfltEnv['hs_multi_client']:
+ hs_client_list = client_list
+ hs_path_node_count *= len(client_list)
+ else:
+ # only use the first client in the list
+ hs_client_list = client_list[:1]
+ # Setup the connections from each client in hs_client_list to each hs
+ for hs in hs_list:
+ hs_bind_to = (hs._env['hs_hostname'], HS_PORT)
+ for client in hs_client_list:
+ print(" HS to %s:%d (%s:%d) via client %s:%s"
+ % (hs._env['hs_hostname'], HS_PORT,
'127.0.0.1', LISTEN_PORT,
'localhost', client._env['socksport']))
- tt.add(chutney.Traffic.Source(tt, bind_to, tmpdata,
+ for i in range(connection_count):
+ tt.add(chutney.Traffic.Source(tt,
+ hs_bind_to,
+ tmpdata,
('localhost',
- int(client._env['socksport']))))
- return tt.run()
+ int(client._env['socksport'])),
+ reps))
+ return hs_path_node_count
+
+ # calculate the single stream bandwidth and overall tor bandwidth
+ # the single stream bandwidth is the bandwidth of the
+ # slowest stream of all the simultaneously transmitted streams
+ # the overall bandwidth estimates the simultaneous bandwidth between
+ # all tor nodes over all simultaneous streams, assuming:
+ # * minimum path lengths (no cannibalized circuits)
+ # * unlimited network bandwidth (that is, localhost)
+ # * tor performance is CPU-limited
+ # This be used to estimate the bandwidth capacity of a CPU-bound
+ # tor relay running on this machine
+ def _report_bandwidth(self, data_length, total_path_node_count,
+ start_time, end_time):
+ # otherwise, if we sent at least 5 MB cumulative total, and
+ # it took us at least a second to send, report bandwidth
+ MIN_BWDATA = 5 * 1024 * 1024 # Octets.
+ MIN_ELAPSED_TIME = 1.0 # Seconds.
+ cumulative_data_sent = total_path_node_count * data_length
+ elapsed_time = end_time - start_time
+ if (cumulative_data_sent >= MIN_BWDATA
+ and elapsed_time >= MIN_ELAPSED_TIME):
+ # Report megabytes per second
+ BWDIVISOR = 1024*1024
+ single_stream_bandwidth = (data_length
+ / elapsed_time
+ / BWDIVISOR)
+ overall_bandwidth = (cumulative_data_sent
+ / elapsed_time
+ / BWDIVISOR)
+ print("Single Stream Bandwidth: %.2f MBytes/s"
+ % single_stream_bandwidth)
+ print("Overall tor Bandwidth: %.2f MBytes/s"
+ % overall_bandwidth)
def ConfigureNodes(nodelist):
diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py
index 46db0f2..3796205 100644
--- a/lib/chutney/Traffic.py
+++ b/lib/chutney/Traffic.py
@@ -141,6 +141,7 @@ class Sink(Peer):
def __init__(self, tt, s):
super(Sink, self).__init__(Peer.SINK, tt, s)
self.inbuf = ''
+ self.repetitions = self.tt.repetitions
def on_readable(self):
"""Invoked when the socket becomes readable.
@@ -151,13 +152,35 @@ class Sink(Peer):
return self.verify(self.tt.data)
def verify(self, data):
+ # shortcut read when we don't ever expect any data
+ if self.repetitions == 0 or len(self.tt.data) == 0:
+ debug("no verification required - no data")
+ return 0;
self.inbuf += self.s.recv(len(data) - len(self.inbuf))
- assert(len(self.inbuf) <= len(data))
- if len(self.inbuf) == len(data):
- if self.inbuf != data:
+ debug("successfully received (bytes=%d)" % len(self.inbuf))
+ while len(self.inbuf) >= len(data):
+ assert(len(self.inbuf) <= len(data) or self.repetitions > 1)
+ if self.inbuf[:len(data)] != data:
+ debug("receive comparison failed (bytes=%d)" % len(data))
return -1 # Failed verification.
+ # if we're not debugging, print a dot every dot_repetitions reps
+ elif (not debug_flag
+ and self.tt.dot_repetitions > 0
+ and self.repetitions % self.tt.dot_repetitions == 0):
+ sys.stdout.write('.')
+ sys.stdout.flush()
+ # repeatedly check data against self.inbuf if required
+ debug("receive comparison success (bytes=%d)" % len(data))
+ self.inbuf = self.inbuf[len(data):]
+ debug("receive leftover bytes (bytes=%d)" % len(self.inbuf))
+ self.repetitions -= 1
+ debug("receive remaining repetitions (reps=%d)" % self.repetitions)
+ if self.repetitions == 0 and len(self.inbuf) == 0:
debug("successful verification")
- return len(data) - len(self.inbuf)
+ # calculate the actual length of data remaining, including reps
+ debug("receive remaining bytes (bytes=%d)"
+ % (self.repetitions*len(data) - len(self.inbuf)))
+ return self.repetitions*len(data) - len(self.inbuf)
class Source(Peer):
@@ -169,13 +192,19 @@ class Source(Peer):
CONNECTING_THROUGH_PROXY = 2
CONNECTED = 5
- def __init__(self, tt, server, buf, proxy=None):
+ def __init__(self, tt, server, buf, proxy=None, repetitions=1):
super(Source, self).__init__(Peer.SOURCE, tt)
self.state = self.NOT_CONNECTED
self.data = buf
self.outbuf = ''
self.inbuf = ''
self.proxy = proxy
+ self.repetitions = repetitions
+ # sanity checks
+ if len(self.data) == 0:
+ self.repetitions = 0
+ if self.repetitions == 0:
+ self.data = {}
self.connect(server)
def connect(self, endpoint):
@@ -200,9 +229,14 @@ class Source(Peer):
debug("proxy handshake successful (fd=%d)" % self.fd())
self.state = self.CONNECTED
self.inbuf = ''
- self.outbuf = self.data
debug("successfully connected (fd=%d)" % self.fd())
- return 1 # Keep us around for writing.
+ # if we have no reps or no data, skip sending actual data
+ if self.want_to_write():
+ return 1 # Keep us around for writing.
+ else:
+ # shortcut write when we don't ever expect any data
+ debug("no connection required - no data")
+ return 0
else:
debug("proxy handshake failed (0x%x)! (fd=%d)" %
(ord(self.inbuf[1]), self.fd()))
@@ -210,10 +244,11 @@ class Source(Peer):
return -1
assert(8 - len(self.inbuf) > 0)
return 8 - len(self.inbuf)
- return 1 # Keep us around for writing.
+ return self.want_to_write() # Keep us around for writing if needed
def want_to_write(self):
- return self.state == self.CONNECTING or len(self.outbuf) > 0
+ return (self.state == self.CONNECTING or len(self.outbuf) > 0
+ or (self.repetitions > 0 and len(self.data) > 0))
def on_writable(self):
"""Invoked when the socket becomes writable.
@@ -224,11 +259,21 @@ class Source(Peer):
if self.state == self.CONNECTING:
if self.proxy is None:
self.state = self.CONNECTED
- self.outbuf = self.data
debug("successfully connected (fd=%d)" % self.fd())
else:
self.state = self.CONNECTING_THROUGH_PROXY
self.outbuf = socks_cmd(self.dest)
+ # we write socks_cmd() to the proxy, then read the response
+ # if we get the correct response, we're CONNECTED
+ if self.state == self.CONNECTED:
+ # repeat self.data into self.outbuf if required
+ if (len(self.outbuf) < len(self.data) and self.repetitions > 0):
+ self.outbuf += self.data
+ self.repetitions -= 1
+ debug("adding more data to send (bytes=%d)" % len(self.data))
+ debug("now have data to send (bytes=%d)" % len(self.outbuf))
+ debug("send repetitions remaining (reps=%d)"
+ % self.repetitions)
try:
n = self.s.send(self.outbuf)
except socket.error as e:
@@ -236,10 +281,19 @@ class Source(Peer):
debug("connection refused (fd=%d)" % self.fd())
return -1
raise
+ # sometimes, this debug statement prints 0
+ # it should print length of the data sent
+ # but the code works regardless of this error
+ debug("successfully sent (bytes=%d)" % n)
self.outbuf = self.outbuf[n:]
if self.state == self.CONNECTING_THROUGH_PROXY:
return 1 # Keep us around.
- return len(self.outbuf) # When 0, we're being removed.
+ debug("bytes remaining on outbuf (bytes=%d)" % len(self.outbuf))
+ # calculate the actual length of data remaining, including reps
+ # When 0, we're being removed.
+ debug("bytes remaining overall (bytes=%d)"
+ % (self.repetitions*len(self.data) + len(self.outbuf)))
+ return self.repetitions*len(self.data) + len(self.outbuf)
class TrafficTester():
@@ -252,12 +306,24 @@ class TrafficTester():
Return True if all tests succeed, else False.
"""
- def __init__(self, endpoint, data={}, timeout=3):
+ def __init__(self,
+ endpoint,
+ data={},
+ timeout=3,
+ repetitions=1,
+ dot_repetitions=0):
self.listener = Listener(self, endpoint)
self.pending_close = []
self.timeout = timeout
self.tests = TestSuite()
self.data = data
+ self.repetitions = repetitions
+ # sanity checks
+ if len(self.data) == 0:
+ self.repetitions = 0
+ if self.repetitions == 0:
+ self.data = {}
+ self.dot_repetitions = dot_repetitions
debug("listener fd=%d" % self.listener.fd())
self.peers = {} # fd:Peer
@@ -318,9 +384,16 @@ class TrafficTester():
self.tests.failure()
self.remove(p)
+ for fd in self.peers:
+ peer = self.peers[fd]
+ debug("peer fd=%d never pending close, never read or wrote" % fd)
+ self.pending_close.append(peer.s)
self.listener.s.close()
for s in self.pending_close:
s.close()
+ if not debug_flag:
+ sys.stdout.write('\n')
+ sys.stdout.flush()
return self.tests.all_done() and self.tests.failure_count() == 0
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits