[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r18207: {torflow} Add the ability to run soat/speedracer runs concurrently wit (torflow/trunk/CircuitAnalysis/BuildTimes)



Author: mikeperry
Date: 2009-01-21 08:52:24 -0500 (Wed, 21 Jan 2009)
New Revision: 18207

Modified:
   torflow/trunk/CircuitAnalysis/BuildTimes/buildtimes.py
   torflow/trunk/CircuitAnalysis/BuildTimes/dist_check.py
   torflow/trunk/CircuitAnalysis/BuildTimes/full_run.sh
Log:

Add the ability to run soat/speedracer runs concurrently with
the buildtimes runs. Rawk. Also add some checks on the router
list rankings. Hit some bug after approx 200k circuits where
the rankings blipped out for a bit (or we stored them wrong,
or python had some braindamage..)



Modified: torflow/trunk/CircuitAnalysis/BuildTimes/buildtimes.py
===================================================================
--- torflow/trunk/CircuitAnalysis/BuildTimes/buildtimes.py	2009-01-21 07:24:50 UTC (rev 18206)
+++ torflow/trunk/CircuitAnalysis/BuildTimes/buildtimes.py	2009-01-21 13:52:24 UTC (rev 18207)
@@ -55,6 +55,7 @@
     self.chosen = [0,0,0]
     self.uptime = 0
  
+# TODO: Make this passive, or make PathBuild have a passive option
 class StatsGatherer(StatsHandler):
   def __init__(self,c, selmgr,basefile_name,nstats):
     StatsHandler.__init__(self,c, selmgr, BTRouter)
@@ -76,6 +77,12 @@
                   ExitPolicyRestriction("255.255.255.255", 80), 
                   ExitPolicyRestriction("255.255.255.255", 443)]))
     self.selmgr.path_selector.exit_gen.rebuild()
+
+
+  def stream_status_event(self, strm_event):
+    # Prevent PathBuilder code from attaching streams
+    # TODO: Passively gather stream stats without building
+    pass
     
   def circ_status_event(self, circ_event):
     """ handles circuit status event """
@@ -292,7 +299,20 @@
   c._handler.schedule_low_prio(notlambda)
   cond.wait()
   cond.release()
-
+  
+  # Write out rank_history and bw listings
+  rankfile = open(basefile_name+".ranks", "w")
+  for r in c._handler.sorted_r:
+    if r.rank_history:
+      rankfile.write("r "+r.idhex+" "+" ".join(map(str, r.rank_history))+"\n")
+    else:
+      rankfile.write("r "+r.idhex+" "+str(r.list_rank)+"\n")
+    if r.bw_history:
+      rankfile.write("b "+r.idhex+" "+" ".join(map(str, r.bw_history))+"\n")
+    else:
+      rankfile.write("b "+r.idhex+" "+str(r.bw)+"\n")
+  rankfile.close()
+ 
   # Run self-checks
   checkfile = open(basefile_name+".check", "w")
   def logger(msg):

Modified: torflow/trunk/CircuitAnalysis/BuildTimes/dist_check.py
===================================================================
--- torflow/trunk/CircuitAnalysis/BuildTimes/dist_check.py	2009-01-21 07:24:50 UTC (rev 18206)
+++ torflow/trunk/CircuitAnalysis/BuildTimes/dist_check.py	2009-01-21 13:52:24 UTC (rev 18207)
@@ -54,19 +54,23 @@
       assert False, "Bad option"
   return pathfile
 
-def check_ranks(r):
-  if not r.rank_history:
-    return (r.list_rank, r.list_rank, r.list_rank)
+
+def min_avg_max(list):
   minr = 100
   maxr = 0
   avgr = 0.0
-  for rank in r.rank_history:
-    avgr += rank
-    if rank < minr: minr = rank
-    if rank > maxr: maxr = rank
-  avgr /= len(r.rank_history)
-  return (avgr,minr,maxr)
+  for v in list:
+    avgr += v
+    if v < minr: minr = v
+    if v > maxr: maxr = v
+  avgr /= len(list)
+  return (minr,avgr,maxr)
 
+def check_ranks(r):
+  if not r.rank_history:
+    return (r.list_rank, r.list_rank, r.list_rank)
+  return min_avg_max(r.rank_history)
+
 def open_controller():
   """ starts stat gathering thread """
   s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
@@ -102,6 +106,7 @@
       router_map[nodes[0]].uptime = float(nodes[1])/60.0
     else:
       total_absent += 1
+  uptimes.close()
 
   pct_mins = [100, 100, 100]
   pct_maxes = [0, 0, 0]
@@ -146,6 +151,29 @@
 
   # FIXME: Compare circuits/chosen to %bw. Multiply by pct_min+max
   # FIXME: Verify by guard+exit weighting?
+  tot_len = len(routers)
+
+  # FIXME: Read in from files, compare against saved infoz
+  rankfile = open(pathfile+".ranks", "r")
+  for line in rankfile:
+    nodes = map(lambda n: n.strip(), line.split(" "))
+    if nodes[0] == 'r': # rank list
+      ranks = map(int, nodes[2:])   
+    elif nodes[0] == 'b': # bw list
+      bws = map(int, nodes[2:])
+    router = router_map[nodes[1]]
+    if router.rank_history:
+      for i,r in enumerate(ranks):
+        if router.rank_history[i] != r:
+          print "WARN: Rank mismatch for "+router.idhex+": "+str(check_ranks(router))+" vs "+str(min_avg_max(ranks))
+          break
+    if router.bw_history:
+      for i,b in enumerate(bws):
+        if router.bw_history[i] != b:
+          print "WARN: Bw mismatch for "+router.idhex+": "+str(check_ranks(router))+" vs "+str(min_avg_max(ranks))
+          break
+  rankfile.close()
+
   for i in xrange(0, 3):
     routers.sort(lambda x, y: cmp(y.chosen[i], x.chosen[i]))
     log("\nHop "+str(i)+": ")
@@ -154,7 +182,7 @@
       if r.chosen[i] == 0: unchosen+=1
       else:
         ranks = check_ranks(r) 
-        log(r.idhex+" "+"/".join(map(lambda f: str(round(f, 2)), ranks))+"%, chosen: "+str(r.chosen[i])+", up: "+str(round(r.uptime,2)))
+        log(r.idhex+" "+("/".join(map(lambda f: str(round(100.0*f/tot_len, 1)), ranks)))+"%\tchosen: "+str(r.chosen[i])+"\tup: "+str(round(r.uptime,1)))
         #log(r.idhex+" "+str(round((100.0*r.list_rank)/len(routers),2))+"%, chosen: "+str(r.chosen[i])+", up: "+str(round(r.uptime,2)))
 
     log("Nodes not chosen for this hop: "+str(unchosen)+"/"+str(len(routers)))

Modified: torflow/trunk/CircuitAnalysis/BuildTimes/full_run.sh
===================================================================
--- torflow/trunk/CircuitAnalysis/BuildTimes/full_run.sh	2009-01-21 07:24:50 UTC (rev 18206)
+++ torflow/trunk/CircuitAnalysis/BuildTimes/full_run.sh	2009-01-21 13:52:24 UTC (rev 18207)
@@ -2,8 +2,8 @@
 
 mkdir slices
 
-./buildtimes.py -n 10000 -s 3 -e 93 -d -c 15 ./slices >& ./slices/bt-slices.log
-./buildtimes.py -n 10000 -s 3 -g -e 50 -d -c 30 ./slices >& ./slices/bt-guards.log
-./buildtimes.py -n 20000 -d slices/ -s 93 -c 100 >& ./slices/bt-all.log
+./buildtimes.py -n 10000 -s 3 -e 93 -c 15 -d ./slices >& ./slices/bt-slices.log
+./buildtimes.py -n 10000 -s 3 -g -e 50 -c 30 -d ./slices >& ./slices/bt-guards.log
+./buildtimes.py -n 100000 -d slices/ -s 93 -c 100 >& ./slices/bt-all.log
 
 mv slices slices-`date +%Y-%m-%d-%H:%M`