[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r19623: {torflow} Directory reorg, minor aggregator fixes, and stats output fi (in torflow/trunk/NetworkScanners/BwAuthority: . data data/scanner.1 data/scanner.2 data/scanner.3)



Author: mikeperry
Date: 2009-06-03 16:18:39 -0400 (Wed, 03 Jun 2009)
New Revision: 19623

Added:
   torflow/trunk/NetworkScanners/BwAuthority/data/
   torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/
   torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
   torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/
   torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
   torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/
   torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
Removed:
   torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
   torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
   torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
   torflow/trunk/NetworkScanners/BwAuthority/scanner.1/
   torflow/trunk/NetworkScanners/BwAuthority/scanner.2/
   torflow/trunk/NetworkScanners/BwAuthority/scanner.3/
Modified:
   torflow/trunk/NetworkScanners/BwAuthority/aggregate.py
   torflow/trunk/NetworkScanners/BwAuthority/bwauthority.py
   torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/torrc
   torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/torrc
   torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/torrc
   torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh
Log:

Directory reorg, minor aggregator fixes, and stats output filter
fix to handle the fact that consensus updates can change our 
percentiles during a scan and cause us to report unmeasured
nodes and not report measured nodes.



Modified: torflow/trunk/NetworkScanners/BwAuthority/aggregate.py
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/aggregate.py	2009-06-03 17:40:13 UTC (rev 19622)
+++ torflow/trunk/NetworkScanners/BwAuthority/aggregate.py	2009-06-03 20:18:39 UTC (rev 19623)
@@ -7,11 +7,11 @@
 bw_files = {}
 nodes = {}
 
-# XXX: Do not generate output file if we don't have a 'done' file
-# from each scanner.
 # XXX: Alpha smoothing, here or in tor?
-# XXX: Consolidate scanner.1, scanner.2, scanner.3 into one dir
-# so it is easier to traverse them.
+#      - Tor, because we want to be able to opine about stuff
+#        we did not measure this round
+#      - Or maybe that's an argument for reading the whole consensus
+#        and doing it here.. but that will always be one behind..
 
 def base10_round(bw_val):
   # This keeps the first 3 decimal digits of the bw value only
@@ -79,23 +79,32 @@
     self.ns_bw = int(re.search("[\s]*ns_bw=([\S]+)[\s]*", line).group(1))
 
 def main(argv):
-  for d in argv[1:-1]:
+  for da in argv[1:-1]:
     # First, create a list of the most recent files in the
     # scan dirs that are recent enough
-    for root, dirs, files in os.walk(d):
-      for f in files:
-        if re.search("^bws-[\S]+-done-", f): 
-          fp = file(d+"/"+f, "r")
-          ranks = fp.readline()
-          timestamp = float(fp.readline())
-          fp.close()
-          if ranks not in bw_files or bw_files[ranks][0] < timestamp:
-            bw_files[ranks] = (timestamp, d+"/"+f)
+    for root, dirs, f in os.walk(da):
+      for ds in dirs:
+        print ds
+        if re.match("^scanner.[\d+]$", ds):
+          print ds
+          for sr, sd, files in os.walk(da+"/"+ds+"/scan-data"):
+            for f in files:
+              if re.search("^bws-[\S]+-done-", f):
+                print sr+"/"+f
+                found_done = True
+                fp = file(sr+"/"+f, "r")
+                slicenum = sr+"/"+fp.readline()
+                timestamp = float(fp.readline())
+                fp.close()
+                if slicenum not in bw_files \
+                       or bw_files[slicenum][0] < timestamp:
+                  bw_files[slicenum] = (timestamp, sr+"/"+f)
+          
   
   for (t,f) in bw_files.itervalues():
     fp = file(f, "r")
-    fp.readline()
-    fp.readline()
+    fp.readline() # slicenum
+    fp.readline() # timestamp
     for l in fp.readlines():
       try:
         line = Line(l)

Modified: torflow/trunk/NetworkScanners/BwAuthority/bwauthority.py
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/bwauthority.py	2009-06-03 17:40:13 UTC (rev 19622)
+++ torflow/trunk/NetworkScanners/BwAuthority/bwauthority.py	2009-06-03 20:18:39 UTC (rev 19623)
@@ -71,6 +71,7 @@
   nodes_per_slice = config.getint('BwAuthority', 'nodes_per_slice')
   save_every = config.getint('BwAuthority', 'save_every')
   circs_per_node = config.getint('BwAuthority', 'circs_per_node')
+  min_streams = config.getint('BwAuthority', 'min_streams')
   out_dir = config.get('BwAuthority', 'out_dir')
   tor_dir = config.get('BwAuthority', 'tor_dir')
   max_fetch_time = config.getint('BwAuthority', 'max_fetch_time')
@@ -83,7 +84,7 @@
 
   return (start_pct,stop_pct,nodes_per_slice,save_every,
             circs_per_node,out_dir,max_fetch_time,tor_dir,
-            sleep_start,sleep_stop)
+            sleep_start,sleep_stop,min_streams)
 
 def choose_url(percentile):
   for (pct, url) in urls:
@@ -106,16 +107,15 @@
     self.add_event_listener(self.sql_consensus_listener)
     self.add_event_listener(SQLSupport.StreamListener())
 
-  def write_sql_stats(self, percent_skip, percent_fast, rfilename=None):
+  def write_sql_stats(self, stats_filter=None, rfilename=None):
     if not rfilename:
       rfilename="./data/stats/sql-"+time.strftime("20%y-%m-%d-%H:%M:%S")
     cond = threading.Condition()
     def notlambda(h):
       cond.acquire()
       SQLSupport.RouterStats.write_stats(file(rfilename, "w"),
-                            percent_skip, percent_fast,
-                            order_by=SQLSupport.RouterStats.sbw,
-                            recompute=True)
+                            0, 100, order_by=SQLSupport.RouterStats.sbw,
+                            recompute=True, stat_clause=stats_filter)
       cond.notify()
       cond.release()
     cond.acquire()
@@ -123,19 +123,17 @@
     cond.wait()
     cond.release()
 
-  def write_strm_bws(self, percent_skip, percent_fast, rfilename=None):
+  def write_strm_bws(self, slice_num, stats_filter=None, rfilename=None):
     if not rfilename:
       rfilename="./data/stats/bws-"+time.strftime("20%y-%m-%d-%H:%M:%S")
     cond = threading.Condition()
     def notlambda(this):
       cond.acquire()
       f=file(rfilename, "w")
-      f.write("low="+str(int(round((percent_skip*len(this.sorted_r))/100.0,0)))
-             +" hi="+str(int(round((percent_fast*len(this.sorted_r))/100.0,0)))
-             +"\n")
-      SQLSupport.RouterStats.write_bws(f, percent_skip, percent_fast,
+      f.write("slicenum="+str(slice_num)+"\n")
+      SQLSupport.RouterStats.write_bws(f, 0, 100,
                             order_by=SQLSupport.RouterStats.sbw,
-                            recompute=False)
+                            recompute=False, stat_clase=stats_filter)
       f.close()
       cond.notify()
       cond.release()
@@ -240,8 +238,9 @@
     def notlambda(this):
       cond.acquire()
       # TODO: Using the entry_gen router list is somewhat ghetto..
-      if this.selmgr.path_selector.entry_gen.rstr_routers and \
-          this.selmgr.path_selector.exit_gen.rstr_routers:
+      if (not this.selmgr.bad_restrictions) or \
+        (this.selmgr.path_selector.entry_gen.rstr_routers and \
+          this.selmgr.path_selector.exit_gen.rstr_routers):
         for r in this.selmgr.path_selector.entry_gen.rstr_routers:
           if r._generated[position] < count:
             cond._finished = False
@@ -381,15 +380,17 @@
       hi = str(round(stop_pct,1))
       # Warning, don't remove the sql stats without changing the recompute
       # param in write_strm_bws to True
-      hdlr.write_sql_stats(start_pct, stop_pct, os.getcwd()+'/'+out_dir+'/sql-'+lo+':'+hi+"-"+str(successful)+"-"+race_time)
-      hdlr.write_strm_bws(start_pct, stop_pct, os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-"+str(successful)+"-"+race_time)
+      hdlr.write_sql_stats(os.getcwd()+'/'+out_dir+'/sql-'+lo+':'+hi+"-"+str(successful)+"-"+race_time, stats_filter=SQLSupport.RouterStats.circ_chosen >= 1)
 
+      hdlr.write_strm_bws(os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-"+str(successful)+"-"+race_time, stats_filter=SQLSupport.RouterStats.strm_chosen >= 1)
+
   plog('INFO', str(start_pct) + '-' + str(stop_pct) + '% ' + str(successful) + ' fetches took ' + str(attempt) + ' tries.')
 
 def main(argv):
   TorUtil.read_config(argv[1])
   (start_pct,stop_pct,nodes_per_slice,save_every,circs_per_node,out_dir,
-      max_fetch_time,tor_dir,sleep_start,sleep_stop) = read_config(argv[1])
+      max_fetch_time,tor_dir,sleep_start,sleep_stop,
+             min_streams) = read_config(argv[1])
  
   try:
     (c,hdlr) = setup_handler(out_dir, tor_dir+"/control_auth_cookie")
@@ -408,7 +409,7 @@
   while True:
     pct = start_pct
     plog('INFO', 'Beginning time loop')
-    
+    slice_num = 0 
     while pct < stop_pct:
       pct_step = hdlr.rank_to_percent(nodes_per_slice)
       hdlr.reset_stats()
@@ -424,14 +425,21 @@
 
       lo = str(round(pct,1))
       hi = str(round(pct+pct_step,1))
-
-      # Warning, don't remove the sql stats without changing the recompute
+      # There may be a consensus change between the point of speed
+      # racing and the writing of stats causing a discrepency
+      # between the immediate, current consensus result used to determine
+      # termination and this average-based result :(
+      # We may need a filter based on circuit chosen and not percentage, or 
+      # we may need to convert this to be most recent rank, and not avg 
+      # (the latter still leaves a small race condition).
+      hdlr.write_sql_stats(os.getcwd()+'/'+out_dir+'/sql-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"), stats_clause=SQLSupport.RouterStats.circ_chosen >= 1)
+      # Warning, don't remove the sql stats call without changing the recompute
       # param in write_strm_bws to True
-      hdlr.write_sql_stats(pct, pct+pct_step, os.getcwd()+'/'+out_dir+'/sql-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"))
-      hdlr.write_strm_bws(pct, pct+pct_step, os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"))
+      hdlr.write_strm_bws(os.getcwd()+'/'+out_dir+'/bws-'+lo+':'+hi+"-done-"+time.strftime("20%y-%m-%d-%H:%M:%S"), slice_num, stats_clause=SQLSupport.RouterStats.strm_closed >= min_streams)
       plog('DEBUG', 'Wrote stats')
       pct += pct_step
       hdlr.save_sql_file(sql_file, os.getcwd()+"/"+out_dir+"/db-"+str(lo)+":"+str(hi)+"-"+time.strftime("20%y-%m-%d-%H:%M:%S")+".sqlite")
+      slice_num += 1
 
 def cleanup(c, f):
   plog("INFO", "Resetting __LeaveStreamsUnattached=0 and FetchUselessDescriptors="+f)


Property changes on: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1
___________________________________________________________________
Added: svn:mergeinfo
   + 

Deleted: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.1/bwauthority.cfg	2009-05-30 22:32:36 UTC (rev 19598)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg	2009-06-03 20:18:39 UTC (rev 19623)
@@ -1,20 +0,0 @@
-[TorCtl]
-loglevel=DEBUG
-tor_host = 127.0.0.1
-tor_port = 9110
-control_host = 127.0.0.1
-control_port = 9111
-control_pass = 
-# XXX: Unused
-meta_host = 127.0.0.1
-meta_port = 9112
-
-[BwAuthority]
-out_dir = ./scanner.1/scan-data
-tor_dir = ./scanner.1/tor-data
-start_pct = 0
-stop_pct = 50
-save_every = 10
-nodes_per_slice = 50
-circs_per_node = 2
-max_fetch_time = 1800

Copied: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg (from rev 19617, torflow/trunk/NetworkScanners/BwAuthority/scanner.1/bwauthority.cfg)
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg	                        (rev 0)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg	2009-06-03 20:18:39 UTC (rev 19623)
@@ -0,0 +1,23 @@
+[TorCtl]
+loglevel=DEBUG
+tor_host = 127.0.0.1
+tor_port = 9110
+control_host = 127.0.0.1
+control_port = 9111
+control_pass = 
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9112
+
+[BwAuthority]
+out_dir = ./data/scanner.1/scan-data
+tor_dir = ./data/scanner.1/tor-data
+start_pct = 0
+stop_pct = 20
+save_every = 10
+nodes_per_slice = 50
+circs_per_node = 2
+min_streams = 1
+max_fetch_time = 1800
+sleep_start = 01:30
+sleep_stop = 05:30

Modified: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/torrc
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.1/torrc	2009-05-30 22:32:36 UTC (rev 19598)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.1/torrc	2009-06-03 20:18:39 UTC (rev 19623)
@@ -2,9 +2,9 @@
 SocksPort 9110 # what port to advertise for application connections
 ControlPort 9111
 SafeLogging 0
-Log notice file ./scanner.1/tor.log
+Log notice file ./data/scanner.1/tor.log
 FetchUselessDescriptors 1
-DataDirectory ./scanner.1/tor-data
+DataDirectory ./data/scanner.1/tor-data
 CookieAuthentication 1
 
 # XXX: This should be for test scans only. Our timeout values must match the


Property changes on: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2
___________________________________________________________________
Added: svn:mergeinfo
   + 

Deleted: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.2/bwauthority.cfg	2009-05-30 22:32:36 UTC (rev 19598)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg	2009-06-03 20:18:39 UTC (rev 19623)
@@ -1,20 +0,0 @@
-[TorCtl]
-loglevel=DEBUG
-tor_host = 127.0.0.1
-tor_port = 9120
-control_host = 127.0.0.1
-control_port = 9121
-control_pass = 
-# XXX: Unused
-meta_host = 127.0.0.1
-meta_port = 9122
-
-[BwAuthority]
-out_dir = ./scanner.2/scan-data
-tor_dir = ./scanner.2/tor-data
-start_pct = 50
-stop_pct = 75
-save_every = 10
-nodes_per_slice = 50
-circs_per_node = 2
-max_fetch_time = 1800

Copied: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg (from rev 19617, torflow/trunk/NetworkScanners/BwAuthority/scanner.2/bwauthority.cfg)
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg	                        (rev 0)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg	2009-06-03 20:18:39 UTC (rev 19623)
@@ -0,0 +1,23 @@
+[TorCtl]
+loglevel=DEBUG
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass = 
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.2/scan-data
+tor_dir = ./data/scanner.2/tor-data
+start_pct = 20
+stop_pct = 60
+save_every = 10
+nodes_per_slice = 50
+circs_per_node = 2
+min_streams = 1
+max_fetch_time = 1800
+sleep_start = 01:30
+sleep_stop = 05:30

Modified: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/torrc
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.2/torrc	2009-05-30 22:32:36 UTC (rev 19598)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.2/torrc	2009-06-03 20:18:39 UTC (rev 19623)
@@ -2,9 +2,9 @@
 SocksPort 9120 # what port to advertise for application connections
 ControlPort 9121
 SafeLogging 0
-Log notice file ./scanner.2/tor.log
+Log notice file ./data/scanner.2/tor.log
 FetchUselessDescriptors 1
-DataDirectory ./scanner.2/tor-data
+DataDirectory ./data/scanner.2/tor-data
 CookieAuthentication 1
 
 # XXX: This should be for test scans only. Our timeout values must match the


Property changes on: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3
___________________________________________________________________
Added: svn:mergeinfo
   + 

Deleted: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.3/bwauthority.cfg	2009-05-30 22:32:36 UTC (rev 19598)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg	2009-06-03 20:18:39 UTC (rev 19623)
@@ -1,20 +0,0 @@
-[TorCtl]
-loglevel=DEBUG
-tor_host = 127.0.0.1
-tor_port = 9130
-control_host = 127.0.0.1
-control_port = 9131
-control_pass = 
-# XXX: Unused
-meta_host = 127.0.0.1
-meta_port = 9132
-
-[BwAuthority]
-out_dir = ./scanner.3/scan-data
-tor_dir = ./scanner.3/tor-data
-start_pct =75
-stop_pct = 100
-save_every = 10
-nodes_per_slice = 50
-circs_per_node = 2
-max_fetch_time = 1800

Copied: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg (from rev 19617, torflow/trunk/NetworkScanners/BwAuthority/scanner.3/bwauthority.cfg)
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg	                        (rev 0)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg	2009-06-03 20:18:39 UTC (rev 19623)
@@ -0,0 +1,23 @@
+[TorCtl]
+loglevel=DEBUG
+tor_host = 127.0.0.1
+tor_port = 9130
+control_host = 127.0.0.1
+control_port = 9131
+control_pass = 
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9132
+
+[BwAuthority]
+out_dir = ./data/scanner.3/scan-data
+tor_dir = ./data/scanner.3/tor-data
+start_pct =60
+stop_pct = 100
+save_every = 10
+nodes_per_slice = 50
+circs_per_node = 2
+min_streams = 1
+max_fetch_time = 1800
+sleep_start = 01:30
+sleep_stop = 05:30

Modified: torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/torrc
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/scanner.3/torrc	2009-05-30 22:32:36 UTC (rev 19598)
+++ torflow/trunk/NetworkScanners/BwAuthority/data/scanner.3/torrc	2009-06-03 20:18:39 UTC (rev 19623)
@@ -2,9 +2,9 @@
 SocksPort 9130 # what port to advertise for application connections
 ControlPort 9131
 SafeLogging 0
-Log notice file ./scanner.3/tor.log
+Log notice file ./data/scanner.3/tor.log
 FetchUselessDescriptors 1
-DataDirectory ./scanner.3/tor-data
+DataDirectory ./data/scanner.3/tor-data
 CookieAuthentication 1
 
 # XXX: This should be for test scans only. Our timeout values must match the

Modified: torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh
===================================================================
--- torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh	2009-06-03 17:40:13 UTC (rev 19622)
+++ torflow/trunk/NetworkScanners/BwAuthority/run_scan.sh	2009-06-03 20:18:39 UTC (rev 19623)
@@ -7,16 +7,16 @@
 #      git branch --track rs-format-fix mikeperry/rs-format-fix
 #      git checkout rs-format-fix
 TOR_EXE=../../../tor.git/src/or/tor
-PYTHONPATH=../../../SQLAlchemy-0.5.4p2/lib
+#PYTHONPATH=../../../SQLAlchemy-0.5.4p2/lib
 
-for i in scanner.*
+for i in data/scanner.*
 do
   rm $i/scan-data/*
 done
 
-$TOR_EXE -f ./scanner.1/torrc & 
-$TOR_EXE -f ./scanner.2/torrc & 
-$TOR_EXE -f ./scanner.3/torrc & 
+$TOR_EXE -f ./data/scanner.1/torrc & 
+$TOR_EXE -f ./data/scanner.2/torrc & 
+$TOR_EXE -f ./data/scanner.3/torrc & 
 
 # If this is a fresh start, we should allow the tors time to download
 # new descriptors.
@@ -24,8 +24,8 @@
 
 export PYTHONPATH
 
-./bwauthority.py ./scanner.1/bwauthority.cfg >& ./scanner.1/bw.log &
-./bwauthority.py ./scanner.2/bwauthority.cfg >& ./scanner.2/bw.log &
-./bwauthority.py ./scanner.3/bwauthority.cfg >& ./scanner.3/bw.log &
+./bwauthority.py ./data/scanner.1/bwauthority.cfg >& ./data/scanner.1/bw.log &
+./bwauthority.py ./data/scanner.2/bwauthority.cfg >& ./data/scanner.2/bw.log &
+./bwauthority.py ./data/scanner.3/bwauthority.cfg >& ./data/scanner.3/bw.log &