[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [torflow/master] Double the scanners, double the tors



commit 94dff967fd457c4922fe7838f5f5ef4c7b17eefe
Author: Tom Ritter <tom@xxxxxxxxx>
Date:   Thu May 21 13:01:07 2015 -0500

    Double the scanners, double the tors
    
    Also update aggregate.py to work with a two-tor system
---
 NetworkScanners/BwAuthority/aggregate.py           |    6 ++--
 .../BwAuthority/data/scanner.1/bwauthority.cfg     |    4 +--
 .../BwAuthority/data/scanner.2/bwauthority.cfg     |    6 ++--
 .../BwAuthority/data/scanner.3/bwauthority.cfg     |    6 ++--
 .../BwAuthority/data/scanner.4/bwauthority.cfg     |    6 ++--
 .../BwAuthority/data/scanner.5/bwauthority.cfg     |   25 ++++++++--------
 .../BwAuthority/data/scanner.6/bwauthority.cfg     |   30 ++++++++++++++++++++
 .../BwAuthority/data/scanner.7/bwauthority.cfg     |   30 ++++++++++++++++++++
 .../BwAuthority/data/scanner.8/bwauthority.cfg     |   30 ++++++++++++++++++++
 .../BwAuthority/data/scanner.9/bwauthority.cfg     |   30 ++++++++++++++++++++
 .../data/scanner.9/scan-data/.gitignore            |    2 ++
 NetworkScanners/BwAuthority/data/tor.1/torrc       |   18 ++++++++++++
 NetworkScanners/BwAuthority/data/tor.2/torrc       |   18 ++++++++++++
 NetworkScanners/BwAuthority/data/tor/torrc         |   18 ------------
 NetworkScanners/BwAuthority/run_scan.sh            |   29 +++++++++++--------
 15 files changed, 203 insertions(+), 55 deletions(-)

diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py
index 49ad3cc..6d35dba 100755
--- a/NetworkScanners/BwAuthority/aggregate.py
+++ b/NetworkScanners/BwAuthority/aggregate.py
@@ -362,7 +362,8 @@ def write_file_list(datadir):
 
 def main(argv):
   TorUtil.read_config(argv[1]+"/scanner.1/bwauthority.cfg")
-  TorUtil.loglevel = "NOTICE"
+  TorUtil.loglevel = "DEBUG"
+  TorUtil.logfile = "data/aggregate-debug.log"
 
   (branch, head) = TorUtil.get_git_version(PATH_TO_TORFLOW_REPO)
   plog('INFO', 'TorFlow Version: %s' % branch+' '+head)
@@ -373,7 +374,7 @@ def main(argv):
   s.connect((TorUtil.control_host,TorUtil.control_port))
   c = TorCtl.Connection(s)
   c.debug(file(argv[1]+"/aggregate-control.log", "w", buffering=0))
-  c.authenticate_cookie(file(argv[1]+"/tor/control_auth_cookie",
+  c.authenticate_cookie(file(argv[1]+"/tor.1/control_auth_cookie",
                          "r"))
 
   ns_list = c.get_network_status()
@@ -847,6 +848,7 @@ def main(argv):
   plog("INFO", "Avg network +pid_error=+"+str(sum(map(lambda n: n.pid_error, p_nodes))/len(p_nodes)))
   plog("INFO", "Avg network -pid_error="+str(sum(map(lambda n: n.pid_error, n_nodes))/len(n_nodes)))
 
+
   plog("INFO",
        "Measured "+str(measured_pct) +"% of all tor nodes ("
        +str(measured_bw_pct)+"% of previous consensus bw).")
diff --git a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
index 7f81a0b..ad9a4cd 100644
--- a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
@@ -16,9 +16,9 @@ pid_file = ./data/scanner.1/bwauthority.pid
 db_url =
 #db_url = mysql+mysqldb://bwscanner:password@xxxxxxxxx/BwScan1
 #db_url = postgresql://bwscanner:password@xxxxxxxxx/BwScan1
-tor_dir = ./data/tor
+tor_dir = ./data/tor.1
 start_pct = 0
-stop_pct = 10
+stop_pct = 12
 save_every = 0
 nodes_per_slice = 50
 circs_per_node = 5
diff --git a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
index 28cfaa6..3e94e5e 100644
--- a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
@@ -16,9 +16,9 @@ pid_file = ./data/scanner.2/bwauthority.pid
 db_url =
 #db_url = mysql+mysqldb://bwscanner:password@xxxxxxxxx/BwScan2
 #db_url = postgresql://bwscanner:password@xxxxxxxxx/BwScan2
-tor_dir = ./data/tor
-start_pct = 10
-stop_pct = 40
+tor_dir = ./data/tor.1
+start_pct = 12
+stop_pct = 25
 save_every = 0
 nodes_per_slice = 50
 circs_per_node = 5
diff --git a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
index 7b71aa8..71c3285 100644
--- a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
@@ -16,9 +16,9 @@ pid_file = ./data/scanner.3/bwauthority.pid
 db_url = 
 #db_url = mysql+mysqldb://bwscanner:password@xxxxxxxxx/BwScan3
 #db_url = postgresql://bwscanner:password@xxxxxxxxx/BwScan3
-tor_dir = ./data/tor
-start_pct = 30
-stop_pct = 60
+tor_dir = ./data/tor.1
+start_pct = 25
+stop_pct = 37
 save_every = 0
 nodes_per_slice = 50
 circs_per_node = 5
diff --git a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg
index 0dfea36..9a676ec 100644
--- a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg
@@ -16,9 +16,9 @@ pid_file = ./data/scanner.4/bwauthority.pid
 db_url = 
 #db_url = mysql+mysqldb://bwscanner:password@xxxxxxxxx/BwScan4
 #db_url = postgresql://bwscanner:password@xxxxxxxxx/BwScan4
-tor_dir = ./data/tor
-start_pct = 60
-stop_pct = 100
+tor_dir = ./data/tor.1
+start_pct = 37
+stop_pct = 50
 save_every = 0
 nodes_per_slice = 50
 circs_per_node = 5
diff --git a/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg
index 468a62e..1acb1a2 100644
--- a/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg
@@ -1,30 +1,31 @@
 [TorCtl]
 loglevel=WARN
 tor_host = 127.0.0.1
-tor_port = 9110
+tor_port = 9120
 control_host = 127.0.0.1
-control_port = 9111
+control_port = 9121
 control_pass = 
 # XXX: Unused
 meta_host = 127.0.0.1
-meta_port = 9112
+meta_port = 9122
 
 [BwAuthority]
 out_dir = ./data/scanner.5/scan-data
 pid_file = ./data/scanner.5/bwauthority.pid
 # if db_url is unset bwauthority will default to sqlite
-db_url = 
-#db_url = mysql+mysqldb://bwscanner:password@xxxxxxxxx/BwScan4
-#db_url = postgresql://bwscanner:password@xxxxxxxxx/BwScan4
-tor_dir = ./data/tor
-start_pct = 0
-stop_pct = 100
+db_url =
+#db_url = mysql+mysqldb://bwscanner:password@xxxxxxxxx/BwScan5
+#db_url = postgresql://bwscanner:password@xxxxxxxxx/BwScan5
+tor_dir = ./data/tor.2
+start_pct = 50
+stop_pct = 62
 save_every = 0
 nodes_per_slice = 50
 circs_per_node = 5
 min_streams = 1
-max_fetch_time = 360
+max_fetch_time = 300
 sleep_start = 01:30
 sleep_stop = 01:30
-only_unmeasured = 1
-min_unmeasured = 5
+only_unmeasured = 0
+min_unmeasured = 0
+
diff --git a/NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg
new file mode 100755
index 0000000..5bc3616
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg
@@ -0,0 +1,30 @@
+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass = 
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.6/scan-data
+pid_file = ./data/scanner.6/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url =
+#db_url = mysql+mysqldb://bwscanner:password@xxxxxxxxx/BwScan6
+#db_url = postgresql://bwscanner:password@xxxxxxxxx/BwScan6
+tor_dir = ./data/tor.2
+start_pct = 62
+stop_pct = 75
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 300
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0
diff --git a/NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg
new file mode 100755
index 0000000..2898b80
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg
@@ -0,0 +1,30 @@
+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass = 
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.7/scan-data
+pid_file = ./data/scanner.7/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url = 
+#db_url = mysql+mysqldb://bwscanner:password@xxxxxxxxx/BwScan7
+#db_url = postgresql://bwscanner:password@xxxxxxxxx/BwScan7
+tor_dir = ./data/tor.2
+start_pct = 75
+stop_pct = 87
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 300
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0
diff --git a/NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg
new file mode 100755
index 0000000..56a5cfa
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg
@@ -0,0 +1,30 @@
+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass = 
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.8/scan-data
+pid_file = ./data/scanner.8/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url = 
+#db_url = mysql+mysqldb://bwscanner:password@xxxxxxxxx/BwScan8
+#db_url = postgresql://bwscanner:password@xxxxxxxxx/BwScan8
+tor_dir = ./data/tor.2
+start_pct = 87
+stop_pct = 100
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 300
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0
diff --git a/NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg
new file mode 100644
index 0000000..7e49dbb
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg
@@ -0,0 +1,30 @@
+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass = 
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.9/scan-data
+pid_file = ./data/scanner.9/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url =
+#db_url = mysql+mysqldb://bwscanner:password@xxxxxxxxx/BwScan9
+#db_url = postgresql://bwscanner:password@xxxxxxxxx/BwScan9
+tor_dir = ./data/tor.2
+start_pct = 0
+stop_pct = 100
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 360
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 1
+min_unmeasured = 5
diff --git a/NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore
new file mode 100644
index 0000000..20f86df
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore
@@ -0,0 +1,2 @@
+bws-*
+sql-*
\ No newline at end of file
diff --git a/NetworkScanners/BwAuthority/data/tor.1/torrc b/NetworkScanners/BwAuthority/data/tor.1/torrc
new file mode 100644
index 0000000..933141e
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/tor.1/torrc
@@ -0,0 +1,18 @@
+## Configuration file for metatroller on alternate prot
+SocksPort 9110 # what port to advertise for application connections
+ControlPort 9111
+SafeLogging 0
+Log notice file ./data/tor.1/tor.log
+DataDirectory ./data/tor.1
+PidFile ./data/tor.1/tor.pid
+CookieAuthentication 1
+FetchUselessDescriptors 1
+__LeaveStreamsUnattached 1
+
+# Bad idea? Too much consensus update activity?
+FetchDirInfoExtraEarly 1
+FetchDirInfoEarly 1
+
+# Use CREATE cells for first hop, so we can still measure CPU overload via 
+# circ failure
+FastFirstHopPK 0
diff --git a/NetworkScanners/BwAuthority/data/tor.2/torrc b/NetworkScanners/BwAuthority/data/tor.2/torrc
new file mode 100755
index 0000000..209781f
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/tor.2/torrc
@@ -0,0 +1,18 @@
+## Configuration file for metatroller on alternate prot
+SocksPort 9120 # what port to advertise for application connections
+ControlPort 9121
+SafeLogging 0
+Log notice file ./data/tor.2/tor.log
+DataDirectory ./data/tor.2
+PidFile ./data/tor.2/tor.pid
+CookieAuthentication 1
+FetchUselessDescriptors 1
+__LeaveStreamsUnattached 1
+
+# Bad idea? Too much consensus update activity?
+FetchDirInfoExtraEarly 1
+FetchDirInfoEarly 1
+
+# Use CREATE cells for first hop, so we can still measure CPU overload via 
+# circ failure
+FastFirstHopPK 0
diff --git a/NetworkScanners/BwAuthority/data/tor/torrc b/NetworkScanners/BwAuthority/data/tor/torrc
deleted file mode 100644
index c64273b..0000000
--- a/NetworkScanners/BwAuthority/data/tor/torrc
+++ /dev/null
@@ -1,18 +0,0 @@
-## Configuration file for metatroller on alternate prot
-SocksPort 9110 # what port to advertise for application connections
-ControlPort 9111
-SafeLogging 0
-Log notice file ./data/tor/tor.log
-DataDirectory ./data/tor
-PidFile ./data/tor/tor.pid
-CookieAuthentication 1
-FetchUselessDescriptors 1
-__LeaveStreamsUnattached 1
-
-# Bad idea? Too much consensus update activity?
-FetchDirInfoExtraEarly 1
-FetchDirInfoEarly 1
-
-# Use CREATE cells for first hop, so we can still measure CPU overload via 
-# circ failure
-FastFirstHopPK 0
diff --git a/NetworkScanners/BwAuthority/run_scan.sh b/NetworkScanners/BwAuthority/run_scan.sh
index 8a71b1d..908b76e 100755
--- a/NetworkScanners/BwAuthority/run_scan.sh
+++ b/NetworkScanners/BwAuthority/run_scan.sh
@@ -1,7 +1,9 @@
 #!/bin/sh
 
-# Number of scanners to run.
-SCANNER_COUNT=5
+# Number of applications to run.
+SCANNERS_PER_TOR_COUNT=4
+TOR_COUNT=2
+SCANNER_COUNT=$(($SCANNERS_PER_TOR_COUNT * $TOR_COUNT + 1))
 
 # This tor must have the w status line fix as well as the stream bw fix
 # Ie git master or 0.2.2.x
@@ -19,15 +21,17 @@ for n in `seq $SCANNER_COUNT`; do
 done
 
 KILLED_TOR=false
-if [ -f "./data/tor/tor.pid" ]; then
-  PID=`cat ./data/tor/tor.pid`
-  if kill -0 "$PID" 2>/dev/null; then # it is a running process and we may send signals to it
-    kill $PID
-    if [ $? -eq 0 ]; then
-      KILLED_TOR=true
+for n in `seq $TOR_COUNT`; do
+  PIDFILE=./data/tor.${n}/tor.pid
+  if [ -f $PIDFILE ]; then
+    if kill -0 `head -1 $PIDFILE` 2>/dev/null; then # it is a running process and we may send signals to it
+  	  kill `head -1 $PIDFILE`
+  	  if [ $? -eq 0 ]; then
+  	    KILLED_TOR=true
+  	  fi
     fi
   fi
-fi
+done
 
 sleep 5
 
@@ -35,9 +39,10 @@ sleep 5
 # A more accurate resume could be implemented in bwauthority.py
 find data/scanner.* -name .svn -prune -o -type f -a ! -name '*-done-*' -a ! -name bwauthority.cfg -a ! -name .gitignore -exec rm {} +
 
-rm -f ./data/tor/tor.log
-
-$TOR_EXE -f ./data/tor/torrc &
+for n in `seq $TOR_COUNT`; do
+	rm -f ./data/tor.${n}/tor.log
+	$TOR_EXE -f ./data/tor.${n}/torrc &
+done
 
 # If this is a fresh start, we should allow the tors time to download
 # new descriptors.



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits