[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r19285: {torflow} Classify some errors better, add an exit-level connect error (torflow/trunk/NetworkScanners)



Author: mikeperry
Date: 2009-04-11 02:07:35 -0400 (Sat, 11 Apr 2009)
New Revision: 19285

Modified:
   torflow/trunk/NetworkScanners/libsoat.py
   torflow/trunk/NetworkScanners/snakeinspector.py
   torflow/trunk/NetworkScanners/soat.py
Log:

Classify some errors better, add an exit-level connect
error counter before marking a real failure, print
some more info via snakeinspector, and give it a 
couple more options.



Modified: torflow/trunk/NetworkScanners/libsoat.py
===================================================================
--- torflow/trunk/NetworkScanners/libsoat.py	2009-04-11 05:06:57 UTC (rev 19284)
+++ torflow/trunk/NetworkScanners/libsoat.py	2009-04-11 06:07:35 UTC (rev 19285)
@@ -11,6 +11,8 @@
 import difflib
 import re
 import copy
+import socket
+import struct
 sys.path.append("./libs")
 from OpenSSL import crypto
 from BeautifulSoup.BeautifulSoup import Tag, SoupStrainer
@@ -65,18 +67,19 @@
 FAILURE_DYNAMIC = "FailureDynamic" 
 FAILURE_COOKIEMISMATCH = "FailureCookieMismatch"
 FAILURE_BADHTTPCODE = "FailureBadHTTPCode"
-FAILURE_MISCEXCEPTION = "FailureMiscException"
 FAILURE_NOEXITCONTENT = "FailureNoExitContent"
 FAILURE_EXITTRUNCATION = "FailureExitTruncation"
 FAILURE_SOCKSERROR = "FailureSocksError"
-FAILURE_HOSTUNREACH = "FailureHostUnreach" # Can also mean DNS issues..
+FAILURE_HOSTUNREACH = "FailureHostUnreach" # aka DNS issue
 FAILURE_NETUNREACH = "FailureNetUnreach"
 FAILURE_EXITPOLICY = "FailureExitPolicy"
 FAILURE_CONNREFUSED = "FailureConnRefused"
-FAILURE_URLERROR = "FailureURLError" # can also mean timeout...
+FAILURE_CONNERROR = "FailureConnError"
+FAILURE_URLERROR = "FailureURLError"
 FAILURE_CRYPTOERROR = "FailureCryptoError"
 FAILURE_TIMEOUT = "FailureTimeout"
 FAILURE_HEADERCHANGE = "FailureHeaderChange"
+FAILURE_MISCEXCEPTION = "FailureMiscException"
 
 # False positive reasons
 FALSEPOSITIVE_HTTPERRORS = "FalsePositiveHTTPErrors"
@@ -88,9 +91,12 @@
 
 class TestResult(object):
   ''' Parent class for all test result classes '''
-  def __init__(self, exit_node, exit_name, site, status, reason=None):
-    self.exit_node = exit_node
-    self.exit_name = exit_name
+  def __init__(self, exit_obj, site, status, reason=None):
+    self.exit_node = exit_obj.idhex
+    self.exit_name = exit_obj.nickname
+    self.exit_ip = exit_obj.ip
+    self.contact = exit_obj.contact
+    self.exit_obj = exit_obj
     self.site = site
     self.timestamp = time.time()
     self.status = status
@@ -101,7 +107,7 @@
     self.verbose=0
     self.from_rescan = False
     self.filename=None
-    self._pickle_revision = 2
+    self._pickle_revision = 4
 
   def depickle_upgrade(self):
     if not "_pickle_revision" in self.__dict__: # upgrade to v0
@@ -111,6 +117,13 @@
     if self._pickle_revision < 2:
       self._pickle_revision = 2
       self.exit_name = "NameNotStored!"
+    if self._pickle_revision < 3:
+      self._pickle_revision = 3
+      self.exit_ip = "\x00\x00\x00\x00"
+      self.exit_obj = None
+    if self._pickle_revision < 4:
+      self._pickle_revision = 4
+      self.contact = None
 
   def _rebase(self, filename, new_data_root):
     if not filename: return filename
@@ -142,7 +155,8 @@
   def __str__(self):
     ret = self.__class__.__name__+" for "+self.site+"\n"
     ret += " Time: "+time.ctime(self.timestamp)+"\n"
-    ret += " Exit: "+self.exit_node+" ("+self.exit_name+")\n"
+    ret += " Exit: "+socket.inet_ntoa(struct.pack(">I",self.exit_ip))+" "+self.exit_node+" ("+self.exit_name+")\n"
+    ret += " Contact: "+str(self.contact)+"\n"  
     ret += " "+str(RESULT_STRINGS[self.status])
     if self.reason:
       ret += " Reason: "+self.reason
@@ -157,9 +171,9 @@
 
 class SSLTestResult(TestResult):
   ''' Represents the result of an openssl test '''
-  def __init__(self, exit_node, exit_name, ssl_site, ssl_file, status, 
+  def __init__(self, exit_obj, ssl_site, ssl_file, status, 
                reason=None, exit_ip=None, exit_cert_pem=None):
-    super(SSLTestResult, self).__init__(exit_node, exit_name, ssl_site, status, reason)
+    super(SSLTestResult, self).__init__(exit_obj, ssl_site, status, reason)
     self.ssl_file = ssl_file
     self.exit_cert = exit_cert_pem # Meh, not that much space
     self.exit_ip = exit_ip
@@ -233,10 +247,10 @@
 
 class HttpTestResult(TestResult):
   ''' Represents the result of a http test '''
-  def __init__(self, exit_node, exit_name, website, status, reason=None, 
+  def __init__(self, exit_obj, website, status, reason=None, 
                sha1sum=None, exit_sha1sum=None, content=None, 
                content_exit=None, content_old=None, sha1sum_old=None):
-    super(HttpTestResult, self).__init__(exit_node, exit_name, website, status, reason)
+    super(HttpTestResult, self).__init__(exit_obj, website, status, reason)
     self.proto = "http"
     self.sha1sum = sha1sum
     self.sha1sum_old = sha1sum_old
@@ -275,9 +289,9 @@
     return ret
 
 class CookieTestResult(TestResult):
-  def __init__(self, exit_node, exit_name, status, reason, plain_cookies, 
+  def __init__(self, exit_obj, status, reason, plain_cookies, 
                tor_cookies):
-    super(CookieTestResult, self).__init__(exit_node, exit_name, "cookies", status)
+    super(CookieTestResult, self).__init__(exit_obj, "cookies", status)
     self.proto = "http"
     self.reason = reason
     self.tor_cookies = tor_cookies
@@ -291,10 +305,10 @@
 
 class JsTestResult(TestResult):
   ''' Represents the result of a JS test '''
-  def __init__(self, exit_node, exit_name, website, status, reason=None, 
+  def __init__(self, exit_obj, website, status, reason=None, 
                content=None, content_exit=None, content_old=None,
                jsdiffer=None):
-    super(JsTestResult, self).__init__(exit_node, exit_name, website, status, reason)
+    super(JsTestResult, self).__init__(exit_obj, website, status, reason)
     self.proto = "http"
     self.content = content
     self.content_exit = content_exit
@@ -355,10 +369,10 @@
 
 class HtmlTestResult(TestResult):
   ''' Represents the result of a http test '''
-  def __init__(self, exit_node, exit_name, website, status, reason=None, 
+  def __init__(self, exit_obj, website, status, reason=None, 
                content=None, content_exit=None, content_old=None, 
                soupdiffer=None, jsdiffer=None):
-    super(HtmlTestResult, self).__init__(exit_node, exit_name, website, status, reason)
+    super(HtmlTestResult, self).__init__(exit_obj, website, status, reason)
     self.proto = "http"
     self.content = content
     self.content_exit = content_exit
@@ -470,38 +484,38 @@
 
 class SSHTestResult(TestResult):
   ''' Represents the result of an ssh test '''
-  def __init__(self, exit_node, exit_name, ssh_site, status):
-    super(SSHTestResult, self).__init__(exit_node, exit_name, ssh_site, status)
+  def __init__(self, exit_obj, ssh_site, status):
+    super(SSHTestResult, self).__init__(exit_obj, ssh_site, status)
     self.proto = "ssh"
 
 class DNSTestResult(TestResult):
   ''' Represents the result of a dns test '''
-  def __init__(self, exit_node, exit_name, dns_site, status):
-    super(DNSTestResult, self).__init__(exit_node, exit_name, dns_site, status)
+  def __init__(self, exit_obj, dns_site, status):
+    super(DNSTestResult, self).__init__(exit_obj, dns_site, status)
     self.proto = "dns"
 
 class DNSRebindTestResult(TestResult):
   ''' Represents the result of a dns rebind test '''
-  def __init__(self, exit_node, exit_name, dns_rebind_site, status):
-    super(DNSRebindTestResult, self).__init__(exit_node, exit_name, dns_rebind_site, status)
+  def __init__(self, exit_obj, dns_rebind_site, status):
+    super(DNSRebindTestResult, self).__init__(exit_obj, dns_rebind_site, status)
     self.proto = "dns"
 
 class SMTPTestResult(TestResult):
   ''' Represents the result of an smtp test '''
-  def __init__(self, exit_node, exit_name, smtp_site, status):
-    super(SMTPTestResult, self).__init__(exit_node, exit_name, smtp_site, status)
+  def __init__(self, exit_obj, smtp_site, status):
+    super(SMTPTestResult, self).__init__(exit_obj, smtp_site, status)
     self.proto = "smtp"
 
 class IMAPTestResult(TestResult):
   ''' Represents the result of an imap test '''
-  def __init__(self, exit_node, exit_name, imap_site, status):
-    super(IMAPTestResult, self).__init__(exit_node, exit_name, imap_site, status)
+  def __init__(self, exit_obj, imap_site, status):
+    super(IMAPTestResult, self).__init__(exit_obj, imap_site, status)
     self.proto = "imap"
 
 class POPTestResult(TestResult):
   ''' Represents the result of a pop test '''
-  def __init__(self, exit_node, exit_name, pop_site, status):
-    super(POPTestResult, self).__init__(exit_node, exit_name, pop_site, status)
+  def __init__(self, exit_obj, pop_site, status):
+    super(POPTestResult, self).__init__(exit_obj, pop_site, status)
     self.proto = "pop"
 
 class DataHandler:

Modified: torflow/trunk/NetworkScanners/snakeinspector.py
===================================================================
--- torflow/trunk/NetworkScanners/snakeinspector.py	2009-04-11 05:06:57 UTC (rev 19284)
+++ torflow/trunk/NetworkScanners/snakeinspector.py	2009-04-11 06:07:35 UTC (rev 19285)
@@ -27,20 +27,24 @@
   print "  --dir <datadir>"
   print "  --file <.result file>"
   print "  --exit <idhex>"
+  print "  --before <timestamp as string>"
+  print "  --after <timestamp as string>"
   print "  --reason <soat failure reason>    # may be repeated"
   print "  --noreason <soat failure reason>  # may be repeated"
   print "  --proto <protocol>"
   print "  --resultfilter <TestResult class name>"
   print "  --statuscode <'Failure' or 'Inconclusive'>"
   print "  --sortby <'proto' or 'url' or 'exit' or 'reason'>"
+  print "  --falsepositives"
   print "  --verbose"
   sys.exit(1)
 
 def getargs(argv):
   try:
-    opts,args = getopt.getopt(argv[1:],"d:f:e:r:vt:p:s:o:n:", 
+    opts,args = getopt.getopt(argv[1:],"d:f:e:r:vt:p:s:o:n:a:b:F", 
              ["dir=", "file=", "exit=", "reason=", "resultfilter=", "proto=", 
-              "verbose", "statuscode=", "sortby=", "noreason="])
+              "verbose", "statuscode=", "sortby=", "noreason=", "after=",
+              "before=", "falsepositives"])
   except getopt.GetoptError,err:
     print str(err)
     usage(argv)
@@ -54,14 +58,19 @@
   verbose=1
   proto=None
   resultfilter=None
+  before = 0xffffffff
+  after = 0
   sortby="proto"
+  falsepositives=False
   for o,a in opts:
     if o == '-d' or o == '--dir':
       use_dir = a
     elif o == '-f' or o == '--file':
       use_file = a
-    elif o == '-e' or o == '--exit': 
-      node = a
+    elif o == '-b' or o == '--before':
+      before = time.mktime(time.strptime(a))
+    elif o == '-a' or o == '--after': 
+      after = time.mktime(time.strptime(a))
     elif o == '-r' or o == '--reason': 
       reasons.append(a)
     elif o == '-r' or o == '--noreason': 
@@ -72,6 +81,8 @@
       resultfilter = a
     elif o == '-p' or o == '--proto':
       proto = a
+    elif o == '-F' or o == '--falsepositives':
+      falsepositives = True
     elif o == '-s' or o == '--sortby': 
       if a not in ["proto", "site", "exit", "reason"]:
         usage(argv)
@@ -81,10 +92,10 @@
         result = int(a)
       except ValueError:
         result = RESULT_CODES[a]
-  return use_dir,use_file,node,reasons,noreasons,result,verbose,resultfilter,proto,sortby
+  return use_dir,use_file,node,reasons,noreasons,result,verbose,resultfilter,proto,sortby,before,after,falsepositives
  
 def main(argv):
-  use_dir,use_file,node,reasons,noreasons,result,verbose,resultfilter,proto,sortby=getargs(argv)
+  use_dir,use_file,node,reasons,noreasons,result,verbose,resultfilter,proto,sortby,before,after,falsepositives=getargs(argv)
   dh = DataHandler(use_dir)
   print dh.data_dir
 
@@ -106,6 +117,8 @@
     r.verbose = verbose
     if r.reason in noreasons: continue
     if reasons and r.reason not in reasons: continue
+    if r.timestamp < after or before < r.timestamp: continue
+    if (falsepositives) ^ r.false_positive: continue
     if (not result or r.status == result) and \
        (not proto or r.proto == proto) and \
        (not resultfilter or r.__class__.__name__ == resultfilter):
@@ -113,7 +126,7 @@
         print r
       except IOError, e:
         traceback.print_exc()
-      except:
+      except Exception, e:
         traceback.print_exc()
       print "\n-----------------------------\n"
 

Modified: torflow/trunk/NetworkScanners/soat.py
===================================================================
--- torflow/trunk/NetworkScanners/soat.py	2009-04-11 05:06:57 UTC (rev 19284)
+++ torflow/trunk/NetworkScanners/soat.py	2009-04-11 06:07:35 UTC (rev 19285)
@@ -143,7 +143,14 @@
   except socket.timeout, e:
     plog("WARN", "Socket timeout for "+address+": "+str(e))
     traceback.print_exc()
-    return (-6.0, None, [], "", e.__class__.__name__+str(e)) 
+    return (-6.0, None, [], "", e.__class__.__name__+str(e))
+  except httplib.BadStatusLine, e:
+    plog('NOTICE', "HTTP Error during request of "+address+": "+str(e))
+    if not e.line: 
+      return (-13.0, None, [], "", e.__class__.__name__+"(None)") 
+    else:
+      traceback.print_exc()
+      return (-666.0, None, [], "", e.__class__.__name__+str(e)) 
   except urllib2.HTTPError, e:
     plog('NOTICE', "HTTP Error during request of "+address+": "+str(e))
     if str(e) == "<urlopen error timed out>": # Yah, super ghetto...
@@ -186,7 +193,7 @@
     self.nodes_to_mark = 0
     self.tests_per_node = num_tests_per_node
     self._reset()
-    self._pickle_revision = 4 # Will increment as fields are added
+    self._pickle_revision = 6 # Will increment as fields are added
 
   def run_test(self): 
     raise NotImplemented()
@@ -213,6 +220,14 @@
     if self._pickle_revision < 4:
       self.connect_fails = {}
       self._pickle_revision = 4
+    if self._pickle_revision < 5:
+      self.dns_fails = {}
+      self._pickle_revision = 5
+    if self._pickle_revision < 6:
+      self.dns_fails_per_exit = self.dns_fails
+      self.timeout_fails_per_exit = self.timeout_fails
+      self.connect_fails_per_exit = {}
+      self._pickle_revision = 6
 
   def refill_targets(self):
     if len(self.targets) < self.min_targets:
@@ -340,17 +355,21 @@
                                      max_connect_fail_pct)
 
   def _reset(self):
-    self.connect_fails = {}
-    self.exit_fails = {}
-    self.successes = {}
-    self.dynamic_fails = {}
     self.results = []
     self.targets = []
     self.tests_run = 0
     self.nodes_marked = 0
-    self.timeout_fails = {}
+    self.run_start = time.time()
+    # These are indexed by idhex
+    self.connect_fails_per_exit = {}
+    self.timeout_fails_per_exit = {}
+    self.dns_fails_per_exit = {}
     self.node_results = {}
-    self.run_start = time.time()
+    # These are indexed by site url:
+    self.connect_fails = {}
+    self.exit_fails = {}
+    self.successes = {}
+    self.dynamic_fails = {}
  
   def rewind(self):
     self._reset()
@@ -390,7 +409,7 @@
     
     plog("INFO", self.proto+" success at "+result.exit_node+". This makes "+str(win_cnt)+"/"+str(self.site_tests(result.site))+" node successes for "+result.site)
 
-  def register_connect_failure(self, result): 
+  def _register_site_connect_failure(self, result): 
     if self.rescan_nodes: result.from_rescan = True
     self.results.append(result)
     datahandler.saveResult(result)
@@ -402,19 +421,62 @@
     err_cnt = len(self.connect_fails[result.site])
 
     plog("ERROR", self.proto+" connection fail of "+result.reason+" at "+result.exit_node+". This makes "+str(err_cnt)+"/"+str(self.site_tests(result.site))+" node failures for "+result.site)
-    
+
+  def register_connect_failure(self, result):
+    if self.rescan_nodes: result.from_rescan = True
+    if result.exit_node not in self.connect_fails_per_exit:
+      self.connect_fails_per_exit[result.exit_node] = 0
+    self.connect_fails_per_exit[result.exit_node] += 1
+
+    c_cnt = self.connect_fails_per_exit[result.exit_node]
+   
+    if c_cnt > num_connfails_per_node:
+      if result.extra_info:
+        result.extra_info = str(result.extra_info) + " count: "+str(c_cnt)
+      else: 
+        result.extra_info = str(c_cnt)
+      self._register_site_connect_failure(result)
+      del self.connect_fails_per_exit[result.exit_node]
+      return TEST_FAILURE
+    else:
+      plog("NOTICE", self.proto+" connect fail at "+result.exit_node+". This makes "+str(c_cnt)+" fails")
+      return TEST_INCONCLUSIVE
+
+  def register_dns_failure(self, result):
+    if self.rescan_nodes: result.from_rescan = True
+    if result.exit_node not in self.dns_fails_per_exit:
+      self.dns_fails_per_exit[result.exit_node] = 0
+    self.dns_fails_per_exit[result.exit_node] += 1
+
+    d_cnt = self.dns_fails_per_exit[result.exit_node]
+   
+    if d_cnt > num_dnsfails_per_node:
+      if result.extra_info:
+        result.extra_info = str(result.extra_info) + " count: "+str(d_cnt)
+      else: 
+        result.extra_info = str(d_cnt)
+      self._register_site_connect_failure(result)
+      del self.dns_fails_per_exit[result.exit_node]
+      return TEST_FAILURE
+    else:
+      plog("NOTICE", self.proto+" dns fail at "+result.exit_node+". This makes "+str(d_cnt)+" fails")
+      return TEST_INCONCLUSIVE
+
   def register_timeout_failure(self, result):
     if self.rescan_nodes: result.from_rescan = True
-    if result.exit_node not in self.timeout_fails:
-      self.timeout_fails[result.exit_node] = 0
-    self.timeout_fails[result.exit_node] += 1
+    if result.exit_node not in self.timeout_fails_per_exit:
+      self.timeout_fails_per_exit[result.exit_node] = 0
+    self.timeout_fails_per_exit[result.exit_node] += 1
 
-    t_cnt = self.timeout_fails[result.exit_node]
+    t_cnt = self.timeout_fails_per_exit[result.exit_node]
    
     if t_cnt > num_timeouts_per_node:
-      result.extra_info = str(t_cnt)
-      self.register_connect_failure(result)
-      del self.timeout_fails[result.exit_node]
+      if result.extra_info:
+        result.extra_info = str(result.extra_info) + " count: "+str(t_cnt)
+      else: 
+        result.extra_info = str(t_cnt)
+      self._register_site_connect_failure(result)
+      del self.timeout_fails_per_exit[result.exit_node]
       return TEST_FAILURE
     else:
       plog("NOTICE", self.proto+" timeout at "+result.exit_node+". This makes "+str(t_cnt)+" timeouts")
@@ -602,7 +664,7 @@
     if tor_cookies != plain_cookies:
       exit_node = metacon.get_exit_node()
       plog("ERROR", "Cookie mismatch at "+exit_node+":\nTor Cookies:"+tor_cookies+"\nPlain Cookies:\n"+plain_cookies)
-      result = CookieTestResult(exit_node,self.node_map[exit_node[1:]].nickname,
+      result = CookieTestResult(self.node_map[exit_node[1:]],
                           TEST_FAILURE, FAILURE_COOKIEMISMATCH, plain_cookies, 
                           tor_cookies)
       if self.rescan_nodes: result.from_rescan = True
@@ -797,7 +859,7 @@
     exit_node = metacon.get_exit_node()
     if exit_node == 0 or exit_node == '0' or not exit_node:
       plog('NOTICE', 'We had no exit node to test, skipping to the next test.')
-      result = HttpTestResult("NoExit", "NotSupplied!", 
+      result = HttpTestResult(None, 
                               address, TEST_INCONCLUSIVE, INCONCLUSIVE_NOEXIT)
       if self.rescan_nodes: result.from_rescan = True
       self.results.append(result)
@@ -825,27 +887,41 @@
         return TEST_INCONCLUSIVE 
 
       if pcode < 0 and type(pcode) == float:
-        if pcode == -2: # "connection not allowed aka ExitPolicy
+        if pcode == -1: # "General socks error"
+          fail_reason = FAILURE_CONNERROR
+        elif pcode == -2: # "connection not allowed aka ExitPolicy
           fail_reason = FAILURE_EXITPOLICY
         elif pcode == -3: # "Net Unreach" ??
           fail_reason = FAILURE_NETUNREACH
         elif pcode == -4: # "Host Unreach" aka RESOLVEFAILED
           fail_reason = FAILURE_HOSTUNREACH
+          result = HttpTestResult(self.node_map[exit_node[1:]],
+                                 address, TEST_FAILURE, fail_reason)
+          return self.register_dns_failure(result)
         elif pcode == -5: # Connection refused
           fail_reason = FAILURE_CONNREFUSED
+          result = HttpTestResult(self.node_map[exit_node[1:]], 
+                              address, TEST_FAILURE, fail_reason)
+          self.register_exit_failure(result)
+          return TEST_FAILURE
         elif pcode == -6: # timeout
           fail_reason = FAILURE_TIMEOUT
-          result = HttpTestResult(exit_node, 
-                                 self.node_map[exit_node[1:]].nickname,
+          result = HttpTestResult(self.node_map[exit_node[1:]],
                                  address, TEST_FAILURE, fail_reason)
           return self.register_timeout_failure(result)
+        elif pcode == -13:
+          fail_reason = FAILURE_NOEXITCONTENT
+          result = HttpTestResult(self.node_map[exit_node[1:]], 
+                              address, TEST_FAILURE, fail_reason)
+          self.register_exit_failure(result)
+          return TEST_FAILURE
         elif pcode == -23: 
           fail_reason = FAILURE_URLERROR
         else:
           fail_reason = FAILURE_MISCEXCEPTION
       else: 
         fail_reason = FAILURE_BADHTTPCODE+str(pcode)
-      result = HttpTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = HttpTestResult(self.node_map[exit_node[1:]], 
                             address, TEST_FAILURE, fail_reason)
       result.extra_info = str(pcontent)
       self.register_connect_failure(result)
@@ -853,7 +929,7 @@
 
     # if we have no content, we had a connection error
     if pcontent == "":
-      result = HttpTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = HttpTestResult(self.node_map[exit_node[1:]], 
                               address, TEST_FAILURE, FAILURE_NOEXITCONTENT)
       self.register_exit_failure(result)
       # Restore cookie jars
@@ -868,7 +944,7 @@
     # compare the content
     # if content matches, everything is ok
     if not hdiffs and psha1sum.hexdigest() == sha1sum.hexdigest():
-      result = HttpTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = HttpTestResult(self.node_map[exit_node[1:]], 
                               address, TEST_SUCCESS)
       self.register_success(result)
       return TEST_SUCCESS
@@ -887,8 +963,7 @@
         exit_content_file = open(DataHandler.uniqueFilename(failed_prefix+'.'+exit_node[1:]+'.content'), 'w')
         exit_content_file.write(pcontent)
         exit_content_file.close()
-        result = HttpTestResult(exit_node,
-                                self.node_map[exit_node[1:]].nickname, 
+        result = HttpTestResult(self.node_map[exit_node[1:]], 
                                 address, TEST_FAILURE, FAILURE_EXITTRUNCATION, 
                                 sha1sum.hexdigest(), psha1sum.hexdigest(), 
                                 content_prefix+".content",
@@ -910,7 +985,7 @@
     
     if not content_new:
       plog("WARN", "Failed to re-frech "+address+" outside of Tor. Did our network fail?")
-      result = HttpTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = HttpTestResult(self.node_map[exit_node[1:]], 
                               address, TEST_INCONCLUSIVE, 
                               INCONCLUSIVE_NOLOCALCONTENT)
       if self.rescan_nodes: result.from_rescan = True
@@ -953,8 +1028,7 @@
       # XXX: We probably should store the header differ + exit headers 
       # for later comparison (ie if the header differ picks up more diffs)
       plog("NOTICE", "Post-refetch header changes for "+address+": \n"+hdiffs)
-      result = HttpTestResult(exit_node,
-                              self.node_map[exit_node[1:]].nickname, 
+      result = HttpTestResult(self.node_map[exit_node[1:]], 
                               address, TEST_FAILURE, FAILURE_HEADERCHANGE)
       result.extra_info = hdiffs
       self.register_dynamic_failure(result)
@@ -964,7 +1038,7 @@
     # compare the node content and the new content
     # if it matches, everything is ok
     if psha1sum.hexdigest() == sha1sum_new.hexdigest():
-      result = HttpTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = HttpTestResult(self.node_map[exit_node[1:]], 
                               address, TEST_SUCCESS)
       self.register_success(result)
       return TEST_SUCCESS
@@ -1009,7 +1083,7 @@
       exit_content_file.write(pcontent)
       exit_content_file.close()
 
-      result = HttpTestResult(exit_node, self.node_map[exit_node[1:]].nickname,
+      result = HttpTestResult(self.node_map[exit_node[1:]],
                               address, TEST_FAILURE, FAILURE_EXITONLY, 
                               sha1sum.hexdigest(), psha1sum.hexdigest(), 
                               content_prefix+".content", exit_content_file.name)
@@ -1020,7 +1094,7 @@
     exit_content_file.write(pcontent)
     exit_content_file.close()
 
-    result = HttpTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+    result = HttpTestResult(self.node_map[exit_node[1:]], 
                             address, TEST_FAILURE, FAILURE_DYNAMIC, 
                             sha1sum_new.hexdigest(), psha1sum.hexdigest(), 
                             content_prefix+".content", exit_content_file.name, 
@@ -1245,7 +1319,7 @@
     has_js_changes = jsdiff.contains_differences(tor_js)
 
     if not has_js_changes:
-      result = JsTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = JsTestResult(self.node_map[exit_node[1:]], 
                             address, TEST_SUCCESS)
       self.register_success(result)
       return TEST_SUCCESS
@@ -1254,7 +1328,7 @@
       exit_content_file.write(tor_js)
       exit_content_file.close()
 
-      result = JsTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = JsTestResult(self.node_map[exit_node[1:]], 
                              address, TEST_FAILURE, FAILURE_DYNAMIC, 
                              content_prefix+".content", exit_content_file.name, 
                              content_prefix+'.content-old',
@@ -1301,7 +1375,7 @@
     # if content matches, everything is ok
     if str(orig_soup) == str(tor_soup):
       plog("INFO", "Successful soup comparison after SHA1 fail for "+address+" via "+exit_node)
-      result = HtmlTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = HtmlTestResult(self.node_map[exit_node[1:]], 
                               address, TEST_SUCCESS)
       self.register_success(result)
 
@@ -1310,7 +1384,7 @@
     content_new = new_html.decode('ascii', 'ignore')
     if not content_new:
       plog("WARN", "Failed to re-frech "+address+" outside of Tor. Did our network fail?")
-      result = HtmlTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = HtmlTestResult(self.node_map[exit_node[1:]], 
                               address, TEST_INCONCLUSIVE, 
                               INCONCLUSIVE_NOLOCALCONTENT)
       if self.rescan_nodes: result.from_rescan = True
@@ -1327,7 +1401,7 @@
       exit_content_file.write(tor_html)
       exit_content_file.close()
 
-      result = HtmlTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = HtmlTestResult(self.node_map[exit_node[1:]], 
                               address, TEST_FAILURE, FAILURE_EXITONLY, 
                               content_prefix+".content", exit_content_file.name)
       self.register_exit_failure(result)
@@ -1386,7 +1460,7 @@
 
     if false_positive:
       plog("NOTICE", "False positive detected for dynamic change at "+address+" via "+exit_node)
-      result = HtmlTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = HtmlTestResult(self.node_map[exit_node[1:]], 
                               address, TEST_SUCCESS)
       self.register_success(result)
       return TEST_SUCCESS
@@ -1402,7 +1476,7 @@
       soupdiff_file = content_prefix+".soupdiff"
     else: soupdiff_file = None
 
-    result = HtmlTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+    result = HtmlTestResult(self.node_map[exit_node[1:]], 
                             address, TEST_FAILURE, FAILURE_DYNAMIC, 
                             content_prefix+".content", exit_content_file.name, 
                             content_prefix+'.content-old',
@@ -1595,28 +1669,49 @@
 
     if not cert:
       if code < 0 and type(code) == float:
-        if code == -2: # "connection not allowed aka ExitPolicy
+        if code == -1: # "General socks error"
+          fail_reason = FAILURE_CONNERROR
+        elif code == -2: # "connection not allowed" aka ExitPolicy
           fail_reason = FAILURE_EXITPOLICY
         elif code == -3: # "Net Unreach" ??
           fail_reason = FAILURE_NETUNREACH
         elif code == -4: # "Host Unreach" aka RESOLVEFAILED
           fail_reason = FAILURE_HOSTUNREACH
+          result = SSLTestResult(self.node_map[exit_node[1:]], address,
+                                ssl_file_name, TEST_FAILURE, fail_reason)
+          return self.register_dns_failure(result)
         elif code == -5: # Connection refused
           fail_reason = FAILURE_CONNREFUSED
+          result = SSLTestResult(self.node_map[exit_node[1:]],
+                       address, ssl_file_name, TEST_FAILURE, fail_reason)
+          self.extra_info=exc
+          self.register_exit_failure(result)
+          return TEST_FAILURE
         elif code == -6: # timeout
           fail_reason = FAILURE_TIMEOUT
-          result = SSLTestResult(exit_node,
-                                self.node_map[exit_node[1:]].nickname, address,
+          result = SSLTestResult(self.node_map[exit_node[1:]], address,
                                 ssl_file_name, TEST_FAILURE, fail_reason)
           return self.register_timeout_failure(result)
+        elif code == -13:
+          fail_reason = FAILURE_NOEXITCONTENT # shouldn't happen here
+          result = SSLTestResult(self.node_map[exit_node[1:]],
+                       address, ssl_file_name, TEST_FAILURE, fail_reason)
+          self.extra_info=exc
+          self.register_exit_failure(result)
+          return TEST_FAILURE
         elif code == -23: 
           fail_reason = FAILURE_CRYPTOERROR
+          result = SSLTestResult(self.node_map[exit_node[1:]],
+                       address, ssl_file_name, TEST_FAILURE, fail_reason)
+          self.extra_info=exc
+          self.register_exit_failure(result)
+          return TEST_FAILURE
         else:
           fail_reason = FAILURE_MISCEXCEPTION
       else:
           fail_reason = FAILURE_MISCEXCEPTION
 
-      result = SSLTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = SSLTestResult(self.node_map[exit_node[1:]], 
                              address, ssl_file_name, TEST_FAILURE, fail_reason) 
       result.extra_info = exc
       self.register_connect_failure(result)
@@ -1626,15 +1721,15 @@
       # get an easily comparable representation of the certs
       cert_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
     except OpenSSL.crypto.Error, e:
-      result = SSLTestResult(exit_node, self.node_map[exit_node[1:]].nickname,
+      result = SSLTestResult(self.node_map[exit_node[1:]],
                    address, ssl_file_name, TEST_FAILURE, FAILURE_CRYPTOERROR)
       self.extra_info=e.__class__.__name__+str(e)
-      self.register_connect_failure(result)
+      self.register_exit_failure(result)
       return TEST_FAILURE
 
     # if certs match, everything is ok
     if ssl_domain.seen_cert(cert_pem):
-      result = SSLTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = SSLTestResult(self.node_map[exit_node[1:]], 
                              address, ssl_file_name, TEST_SUCCESS)
       self.register_success(result)
       return TEST_SUCCESS
@@ -1642,7 +1737,7 @@
     # False positive case.. Can't help it if the cert rotates AND we have a
     # failure... Need to prune all results for this cert and give up.
     if ssl_domain.cert_rotates:
-      result = SSLTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+      result = SSLTestResult(self.node_map[exit_node[1:]], 
                              address, ssl_file_name, TEST_FAILURE, 
                              FAILURE_DYNAMIC, self.get_resolved_ip(address), 
                              cert_pem)
@@ -1650,7 +1745,7 @@
       return TEST_FAILURE
 
     # if certs dont match, means the exit node has been messing with the cert
-    result = SSLTestResult(exit_node, self.node_map[exit_node[1:]].nickname, 
+    result = SSLTestResult(self.node_map[exit_node[1:]], 
                            address, ssl_file_name, TEST_FAILURE,
                            FAILURE_EXITONLY, self.get_resolved_ip(address), 
                            cert_pem)
@@ -1835,11 +1930,11 @@
     # compare
     if (capabilities_ok != capabilities_ok_d or starttls_present != starttls_present_d or 
         tls_started != tls_started_d or tls_succeeded != tls_succeeded_d):
-      result = POPTestResult(exit_node, self.node_map[exit_node[1:]].nickname, address, TEST_FAILURE)
+      result = POPTestResult(self.node_map[exit_node[1:]], address, TEST_FAILURE)
       datahandler.saveResult(result)
       return TEST_FAILURE
     
-    result = POPTestResult(exit_node, self.node_map[exit_node[1:]].nickname, address, TEST_SUCCESS)
+    result = POPTestResult(self.node_map[exit_node[1:]], address, TEST_SUCCESS)
     datahandler.saveResult(result)
     return TEST_SUCCESS
 
@@ -1934,11 +2029,11 @@
 
     # compare
     if ehlo1_reply != ehlo1_reply_d or has_starttls != has_starttls_d or ehlo2_reply != ehlo2_reply_d:
-      result = SMTPTestResult(exit_node, self.node_map[exit_node[1:]].nickname, address, TEST_FAILURE)
+      result = SMTPTestResult(self.node_map[exit_node[1:]], address, TEST_FAILURE)
       datahandler.saveResult(result)
       return TEST_FAILURE
 
-    result = SMTPTestResult(exit_node, self.node_map[exit_node[1:]].nickname, address, TEST_SUCCESS)
+    result = SMTPTestResult(self.node_map[exit_node[1:]], address, TEST_SUCCESS)
     datahandler.saveResult(result)
     return TEST_SUCCESS
 
@@ -2101,11 +2196,11 @@
     # compare
     if (capabilities_ok != capabilities_ok_d or starttls_present != starttls_present_d or 
       tls_started != tls_started_d or tls_succeeded != tls_succeeded_d):
-      result = IMAPTestResult(exit_node, self.node_map[exit_node[1:]].nickname, address, TEST_FAILURE)
+      result = IMAPTestResult(self.node_map[exit_node[1:]], address, TEST_FAILURE)
       datahandler.saveResult(result)
       return TEST_FAILURE
 
-    result = IMAPTestResult(exit_node, self.node_map[exit_node[1:]].nickname, address, TEST_SUCCESS)
+    result = IMAPTestResult(self.node_map[exit_node[1:]], address, TEST_SUCCESS)
     datahandler.saveResult(result)
     return TEST_SUCCESS
 
@@ -2136,11 +2231,11 @@
       return TEST_INCONCLUSIVE
 
     if ip in ips_d:
-      result = DNSTestResult(exit_node, self.node_map[exit_node[1:]].nickname, address, TEST_SUCCESS)
+      result = DNSTestResult(self.node_map[exit_node[1:]], address, TEST_SUCCESS)
       return TEST_SUCCESS
     else:
       plog('ERROR', 'The basic DNS test suspects ' + exit_node + ' to be malicious.')
-      result = DNSTestResult(exit_node, self.node_map[exit_node[1:]].nickname, address, TEST_FAILURE)
+      result = DNSTestResult(self.node_map[exit_node[1:]], address, TEST_FAILURE)
       return TEST_FAILURE
 
 class SSHTest(Test):
@@ -2189,6 +2284,16 @@
     c.set_events([TorCtl.EVENT_TYPE.NEWCONSENSUS,
                   TorCtl.EVENT_TYPE.NEWDESC], True)
 
+  def idhex_to_r(self, idhex):
+    self.rlock.acquire()
+    result = None
+    try:
+      if idhex in self.routers:
+        result = self.routers[idhex]
+    finally:
+      self.rlock.release()
+    return result
+
   def name_to_idhex(self, nick):
     self.rlock.acquire()
     result = None
@@ -2267,7 +2372,9 @@
           handler = DataHandler()
           node = self.__mt.get_exit_node()
           plog("ERROR", "DNS Rebeind failure via "+node)
-          result = DNSRebindTestResult(node, "NotStored!", '', TEST_FAILURE)
+
+          result = DNSRebindTestResult(self.__mt.node_manager.idhex_to_r(node), 
+                                       '', TEST_FAILURE)
           handler.saveResult(result)
     # TODO: This is currently handled via socks error codes,
     # but stream events would give us more info...
@@ -2732,8 +2839,16 @@
         test.remove_false_positives()
         if not do_rescan and rescan_at_finish:
           test.toggle_rescan()
-        test.rewind()
-    
+          test.rewind()
+        elif restart_at_finish:
+          test.rewind()
+    all_finished = True
+    for test in tests.itervalues():
+      if not test.finished():
+        all_finished = False
+    if all_finished:
+      plog("NOTICE", "All tests have finished. Exiting\n")
+      sys.exit(0)
 
 # initiate the program
 #