[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] r17957: {torflow} Update speedracer to spit out stats periodically during scan (in torflow/trunk: . CircuitAnalysis/BuildTimes NetworkScanners)
Author: mikeperry
Date: 2009-01-06 11:02:59 -0500 (Tue, 06 Jan 2009)
New Revision: 17957
Added:
torflow/trunk/NetworkScanners/statsplitter.py
Removed:
torflow/trunk/statsplitter.py
Modified:
torflow/trunk/CircuitAnalysis/BuildTimes/buildtimes.py
torflow/trunk/CircuitAnalysis/BuildTimes/shufflebt.py
torflow/trunk/NetworkScanners/speedracer.py
Log:
Update speedracer to spit out stats periodically during
scans. Also move statssplitter and add some comments to the
buildtimes scripts.
Modified: torflow/trunk/CircuitAnalysis/BuildTimes/buildtimes.py
===================================================================
--- torflow/trunk/CircuitAnalysis/BuildTimes/buildtimes.py 2009-01-06 15:55:13 UTC (rev 17956)
+++ torflow/trunk/CircuitAnalysis/BuildTimes/buildtimes.py 2009-01-06 16:02:59 UTC (rev 17957)
@@ -178,9 +178,6 @@
for i in xrange(0,ncircuits):
print 'Building circuit',i
try:
- # XXX: hrmm.. race conditions on the path_selectior members
- # for the event handler thread?
- # Probably only if streams end up coming in during this test..
def notlambda(h):
circ = h.c.build_circuit(h.selmgr.pathlen, h.selmgr.path_selector)
h.circuits[circ.circ_id] = circ
Modified: torflow/trunk/CircuitAnalysis/BuildTimes/shufflebt.py
===================================================================
--- torflow/trunk/CircuitAnalysis/BuildTimes/shufflebt.py 2009-01-06 15:55:13 UTC (rev 17956)
+++ torflow/trunk/CircuitAnalysis/BuildTimes/shufflebt.py 2009-01-06 16:02:59 UTC (rev 17957)
@@ -303,6 +303,8 @@
stddev = s.stddev()
median = s.median()
mode = s.mode() # relies on s.makehistogram for buckets
+
+ # XXX: Try EZfit and/or frechet function
parK = s.paretoK(mode)
modeN = s.modeN(mode)
modeMean = s.modeMean(mode)
@@ -320,7 +322,8 @@
if graph:
# plot histogram
# args: values, # bins, normalize y/n, width of bars
- pylab.hist(s.values,len(s.values) / res, normed=True,width=5)
+ pylab.hist(s.values,(max(s.values)-min(s.values))/res,
+ normed=True,width=5)
#plot Pareto curve
X = pylab.arange(mode, max(s.values), 1)
Modified: torflow/trunk/NetworkScanners/speedracer.py
===================================================================
--- torflow/trunk/NetworkScanners/speedracer.py 2009-01-06 15:55:13 UTC (rev 17956)
+++ torflow/trunk/NetworkScanners/speedracer.py 2009-01-06 16:02:59 UTC (rev 17957)
@@ -24,11 +24,12 @@
# Some constants for measurements
user_agent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)"
-url = "https://svn.torproject.org/svn/tor/trunk/doc/design-paper/tor-design.pdf";
-count = 2
+url = "http://svn.torproject.org/svn/tor/trunk/doc/design-paper/tor-design.pdf";
+count = 250
+save_every = 50
start_pct = 0
-stop_pct = 20
-pct_step = 5
+stop_pct = 78
+pct_step = 3
class MetatrollerException(Exception):
"Metatroller does not accept this command."
@@ -112,6 +113,9 @@
fetch_exit = build_exit
plog('DEBUG', 'circuit build+fetch took ' + str(delta_build) + ' for ' + str(fetch_exit))
+
+ if (successful % save_every) == 0:
+ meta.send_command_and_check('SAVESTATS ./data/speedraces/stats-'+str(pct)+':'+str(pct+pct_step)+"-"+str(successful)+"-"+time.strftime("20%y-%m-%d-%H:%M:%S"))
plog('INFO', str(skip) + '-' + str(pct) + '% ' + str(count) + ' fetches took ' + str(attempt) + ' tries.')
@@ -154,7 +158,7 @@
plog('DEBUG', 'speedroced')
meta.send_command_and_check('CLOSEALLCIRCS')
- meta.send_command_and_check('SAVESTATS ./data/speedraces/stats-' + str(pct) + ':' + str(pct + pct_step))
+ meta.send_command_and_check('SAVESTATS ./data/speedraces/stats-'+str(pct) + ':' + str(pct + pct_step)+"-"+str(count)+"-"+time.strftime("20%y-%m-%d-%H:%M:%S"))
plog('DEBUG', 'Wrote stats')
pct += pct_step
meta.send_command_and_check('COMMIT')
Copied: torflow/trunk/NetworkScanners/statsplitter.py (from rev 17873, torflow/trunk/statsplitter.py)
===================================================================
--- torflow/trunk/NetworkScanners/statsplitter.py (rev 0)
+++ torflow/trunk/NetworkScanners/statsplitter.py 2009-01-06 16:02:59 UTC (rev 17957)
@@ -0,0 +1,113 @@
+#!/usr/bin/python
+import sys
+import socket
+import math
+
+sys.path.append("../")
+#from TorCtl import *
+from TorCtl import TorUtil, PathSupport, TorCtl
+from TorCtl.TorUtil import control_port, control_host
+from TorCtl.TorUtil import *
+from TorCtl.PathSupport import *
+
+TorUtil.loglevel = "NOTICE"
+
+s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+s.connect((control_host,control_port))
+c = Connection(s)
+c.debug(file("control.log", "w"))
+c.authenticate()
+nslist = c.get_network_status()
+sorted_rlist = c.read_routers(c.get_network_status())
+
+sorted_rlist.sort(lambda x, y: cmp(y.bw, x.bw))
+for i in xrange(len(sorted_rlist)): sorted_rlist[i].list_rank = i
+
+fast_rst = FlagsRestriction(["Fast"], [])
+exit_rst = FlagsRestriction(["Exit"], [])
+dir_rst = FlagsRestriction(["V2Dir"], [])
+heavy_exits = OrNodeRestriction(
+ [ExitPolicyRestriction("255.255.255.255", 6881),
+ ExitPolicyRestriction("255.255.255.255", 6889),
+ ExitPolicyRestriction("255.255.255.255", 6346),
+ ExitPolicyRestriction("255.255.255.255", 25)])
+
+
+def check(start, stop):
+ pct_rst = PercentileRestriction(start, stop, sorted_rlist)
+ bw = 0
+ nodes = 0
+ exits = 0
+ exit_bw = 0
+ heavy = 0
+ dirs = 0
+ nodes_up = 0
+ up = 0
+
+ for r in sorted_rlist:
+ if pct_rst.r_is_ok(r) and fast_rst.r_is_ok(r):
+ nodes += 1
+ bw += r.bw
+ if r.uptime > 0:
+ nodes_up += 1.0
+ up += r.uptime
+ if exit_rst.r_is_ok(r):
+ exits += 1
+ exit_bw += r.bw
+ if heavy_exits.r_is_ok(r):
+ heavy += 1
+ if dir_rst.r_is_ok(r):
+ dirs += 1
+
+ print str(start)+"-"+str(stop)+": N: "+str(nodes)+", Bw: "+str(round(bw/(1024*1024.0), 2))+", X: "+str(exits)+", XBw: "+str(round(exit_bw/(1024*1024.0),2))+", BT: "+str(heavy)+", Dirs:"+str(dirs)+", Up: "+str(round(up/nodes_up/60/60/24, 2))
+
+for i in xrange(0,80,3):
+ check(i,i+3)
+
+def check_entropy(rlist, clipping_point):
+ clipped = 0
+ clipped_bw = 0.0
+ exits = 0
+ nodes = 0
+ bw = 0.0
+ exit_bw = 0.0
+ pure_entropy = 0.0
+ clipped_entropy = 0.0
+ uniform_entropy = 0.0
+ for r in rlist:
+ if not fast_rst.r_is_ok(r):
+ continue
+ if r.bw > clipping_point:
+ clipped += 1
+ clipped_bw += clipping_point
+ else:
+ clipped_bw += r.bw
+ nodes += 1
+ bw += r.bw
+ if exit_rst.r_is_ok(r):
+ exits += 1
+ exit_bw += r.bw
+
+ for r in rlist:
+ if not fast_rst.r_is_ok(r):
+ continue
+ if r.bw < 2:
+ continue
+ pure_entropy += (r.bw/bw)*math.log(r.bw/bw, 2)
+ uniform_entropy += (1.0/nodes)*math.log(1.0/nodes, 2)
+
+ rbw = 0
+ if r.bw > clipping_point:
+ rbw = clipping_point
+ else:
+ rbw = r.bw
+ clipped_entropy += (rbw/clipped_bw)*math.log(rbw/clipped_bw, 2)
+
+ print "Uniform entropy: " + str(-uniform_entropy)
+ print "Raw entropy: " + str(-pure_entropy)
+ print "Clipped entropy: " + str(-clipped_entropy)
+ print "Nodes: "+str(nodes)+", Exits: "+str(exits)+" Total bw: "+str(round(bw/(1024.0*1024),2))+", Exit Bw: "+str(round(exit_bw/(1024.0*1024),2))
+ print "Clipped: "+str(clipped)+", bw: "+str(round(clipped_bw/(1024.0*1024),2))
+
+
+check_entropy(sorted_rlist, 1500000)
Deleted: torflow/trunk/statsplitter.py
===================================================================
--- torflow/trunk/statsplitter.py 2009-01-06 15:55:13 UTC (rev 17956)
+++ torflow/trunk/statsplitter.py 2009-01-06 16:02:59 UTC (rev 17957)
@@ -1,111 +0,0 @@
-#!/usr/bin/python
-import sys
-import socket
-import math
-#from TorCtl import *
-from TorCtl import TorUtil, PathSupport, TorCtl
-from TorCtl.TorUtil import control_port, control_host
-from TorCtl.TorUtil import *
-from TorCtl.PathSupport import *
-
-TorUtil.loglevel = "NOTICE"
-
-s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-s.connect((control_host,control_port))
-c = Connection(s)
-c.debug(file("control.log", "w"))
-c.authenticate()
-nslist = c.get_network_status()
-sorted_rlist = c.read_routers(c.get_network_status())
-
-sorted_rlist.sort(lambda x, y: cmp(y.bw, x.bw))
-for i in xrange(len(sorted_rlist)): sorted_rlist[i].list_rank = i
-
-fast_rst = FlagsRestriction(["Fast"], [])
-exit_rst = FlagsRestriction(["Exit"], [])
-dir_rst = FlagsRestriction(["V2Dir"], [])
-heavy_exits = OrNodeRestriction(
- [ExitPolicyRestriction("255.255.255.255", 6881),
- ExitPolicyRestriction("255.255.255.255", 6889),
- ExitPolicyRestriction("255.255.255.255", 6346),
- ExitPolicyRestriction("255.255.255.255", 25)])
-
-
-def check(start, stop):
- pct_rst = PercentileRestriction(start, stop, sorted_rlist)
- bw = 0
- nodes = 0
- exits = 0
- exit_bw = 0
- heavy = 0
- dirs = 0
- nodes_up = 0
- up = 0
-
- for r in sorted_rlist:
- if pct_rst.r_is_ok(r):
- nodes += 1
- bw += r.bw
- if r.uptime > 0:
- nodes_up += 1.0
- up += r.uptime
- if exit_rst.r_is_ok(r):
- exits += 1
- exit_bw += r.bw
- if heavy_exits.r_is_ok(r):
- heavy += 1
- if dir_rst.r_is_ok(r):
- dirs += 1
-
- print str(start)+"-"+str(stop)+": N: "+str(nodes)+", Bw: "+str(round(bw/(1024*1024.0), 2))+", X: "+str(exits)+", XBw: "+str(round(exit_bw/(1024*1024.0),2))+", BT: "+str(heavy)+", Dirs:"+str(dirs)+", Up: "+str(round(up/nodes_up/60/60/24, 2))
-
-for i in xrange(0,80,5):
- check(i,i+5)
-
-def check_entropy(rlist, clipping_point):
- clipped = 0
- clipped_bw = 0.0
- exits = 0
- nodes = 0
- bw = 0.0
- exit_bw = 0.0
- pure_entropy = 0.0
- clipped_entropy = 0.0
- uniform_entropy = 0.0
- for r in rlist:
- if not fast_rst.r_is_ok(r):
- continue
- if r.bw > clipping_point:
- clipped += 1
- clipped_bw += clipping_point
- else:
- clipped_bw += r.bw
- nodes += 1
- bw += r.bw
- if exit_rst.r_is_ok(r):
- exits += 1
- exit_bw += r.bw
-
- for r in rlist:
- if not fast_rst.r_is_ok(r):
- continue
- if r.bw < 2:
- continue
- pure_entropy += (r.bw/bw)*math.log(r.bw/bw, 2)
- uniform_entropy += (1.0/nodes)*math.log(1.0/nodes, 2)
-
- rbw = 0
- if r.bw > clipping_point:
- rbw = clipping_point
- else:
- rbw = r.bw
- clipped_entropy += (rbw/clipped_bw)*math.log(rbw/clipped_bw, 2)
-
- print "Uniform entropy: " + str(-uniform_entropy)
- print "Raw entropy: " + str(-pure_entropy)
- print "Clipped entropy: " + str(-clipped_entropy)
- print "Nodes: "+str(nodes)+", Exits: "+str(exits)+" Total bw: "+str(round(bw/(1024.0*1024),2))+", Exit Bw: "+str(round(exit_bw/(1024.0*1024),2))
- print "Clipped: "+str(clipped)+", bw: "+str(round(clipped_bw/(1024.0*1024),2))
-
-
-check_entropy(sorted_rlist, 1500000)