[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r18443: {torflow} Add initial cut at a snakeinspector.py to dump specific resu (torflow/trunk/NetworkScanners)



Author: mikeperry
Date: 2009-02-09 05:36:42 -0500 (Mon, 09 Feb 2009)
New Revision: 18443

Added:
   torflow/trunk/NetworkScanners/snakeinspector.py
Modified:
   torflow/trunk/NetworkScanners/README.ExitScanning
   torflow/trunk/NetworkScanners/libsoat.py
   torflow/trunk/NetworkScanners/soat.py
Log:

Add initial cut at a snakeinspector.py to dump specific
results in a more manageable format.



Modified: torflow/trunk/NetworkScanners/README.ExitScanning
===================================================================
--- torflow/trunk/NetworkScanners/README.ExitScanning	2009-02-09 09:56:43 UTC (rev 18442)
+++ torflow/trunk/NetworkScanners/README.ExitScanning	2009-02-09 10:36:42 UTC (rev 18443)
@@ -17,6 +17,7 @@
 Python 2.4+
 Custom patched Tor 0.2.1
 Super Secret SoaT Sauce
+py-openssl/pyOpenSSL
 Bonus: 500M of disk space
 Bonus: Secondary external IP address
 
@@ -95,7 +96,8 @@
 IV. Running Tor, The Metatroller, and SoaT
 
 Once you have everything compiled and configured, you should be ready to
-run the pieces. 
+run the pieces. You probably want to do this as a separate, unprivileged
+user.
 
 First, start up your custom Tor with the sample torrc provided in the
 TorFlow svn root:

Modified: torflow/trunk/NetworkScanners/libsoat.py
===================================================================
--- torflow/trunk/NetworkScanners/libsoat.py	2009-02-09 09:56:43 UTC (rev 18442)
+++ torflow/trunk/NetworkScanners/libsoat.py	2009-02-09 10:36:42 UTC (rev 18443)
@@ -10,13 +10,13 @@
 import time
 import traceback
 sys.path.append("./libs")
-from BeautifulSoup.BeautifulSoup import BeautifulSoup, Tag
+from BeautifulSoup.BeautifulSoup import Tag
 
 
 import sets
 from sets import Set
+from soat_config import *
 
-from soat_config import *
 sys.path.append("../")
 from TorCtl.TorUtil import *
 
@@ -34,6 +34,9 @@
 TEST_INCONCLUSIVE = 1
 TEST_FAILURE = 2
 
+# Sorry, we sort of rely on the ordinal nature of the above constants
+RESULT_STRINGS = {TEST_SUCCESS:"Success", TEST_INCONCLUSIVE:"Inconclusive", TEST_FAILURE:"Failure"}
+
 # Inconclusive reasons
 INCONCLUSIVE_NOEXITCONTENT = "InconclusiveNoExitContent"
 INCONCLUSIVE_NOLOCALCONTENT = "InconclusiveNoLocalContent"
@@ -55,12 +58,14 @@
 
 class TestResult(object):
   ''' Parent class for all test result classes '''
-  def __init__(self, exit_node, site, status):
+  def __init__(self, exit_node, site, status, reason=None):
     self.exit_node = exit_node
     self.site = site
     self.timestamp = time.time()
     self.status = status
+    self.reason = reason
     self.false_positive=False
+    self.false_positive_reason="None"
   
   def mark_false_positive(self, reason):
     pass
@@ -77,6 +82,18 @@
       plog("WARN", "Error moving "+file+" to "+dir)
       return file
 
+  def __str__(self):
+    ret = self.__class__.__name__+" for "+self.site+"\n"
+    ret += " Time: "+time.ctime(self.timestamp)+"\n"
+    ret += " Exit: "+self.exit_node+"\n"
+    ret += " "+str(RESULT_STRINGS[self.status])
+    if self.reason:
+      ret += " Reason: "+self.reason
+    if self.false_positive:
+      ret += " (False positive: "+self.false_positive_reason+")"
+    ret += "\n"
+    return ret
+
 class SSLTestResult(TestResult):
   ''' Represents the result of an openssl test '''
   def __init__(self, exit_node, ssl_site, cert_file, status):
@@ -110,17 +127,14 @@
   def __init__(self, exit_node, website, status, reason=None, 
                sha1sum=None, exit_sha1sum=None, content=None, 
                content_exit=None, content_old=None, sha1sum_old=None):
-    super(HttpTestResult, self).__init__(exit_node, website, status)
+    super(HttpTestResult, self).__init__(exit_node, website, status, reason)
     self.proto = "http"
-    self.reason = reason
     self.sha1sum = sha1sum
     self.sha1sum_old = sha1sum_old
     self.exit_sha1sum = exit_sha1sum
     self.content = content
     self.content_exit = content_exit
     self.content_old = content_old
-    self.false_positive=False
-    self.false_positive_reason="None"
 
   def mark_false_positive(self, reason):
     self.false_positive=True
@@ -137,6 +151,16 @@
     try: os.unlink(self.content_exit)
     except: pass
 
+  def __str__(self):
+    ret = TestResult.__str__(self)
+    if self.content:
+      ret += " "+self.content+" (SHA1: "+self.sha1sum+")\n"
+    if self.content_old:
+      ret += " "+self.content_old+" (SHA1: "+self.sha1sum_old+")\n"
+    if self.content_exit:
+      ret += " "+self.content_exit+" (SHA1: "+self.exit_sha1sum+")\n"
+    return ret
+
 class CookieTestResult(TestResult):
   def __init__(self, exit_node, status, reason, plain_cookies, 
                tor_cookies):
@@ -150,14 +174,11 @@
   ''' Represents the result of a JS test '''
   def __init__(self, exit_node, website, status, reason=None, 
                content=None, content_exit=None, content_old=None):
-    super(JsTestResult, self).__init__(exit_node, website, status)
+    super(JsTestResult, self).__init__(exit_node, website, status, reason)
     self.proto = "http"
-    self.reason = reason
     self.content = content
     self.content_exit = content_exit
     self.content_old = content_old
-    self.false_positive=False
-    self.false_positive_reason="None"
 
   def mark_false_positive(self, reason):
     self.false_positive=True
@@ -174,18 +195,26 @@
     try: os.unlink(self.content_exit)
     except: pass
 
+  def __str__(self):
+    # XXX: Re-run the JSDiffer and compare these differences
+    ret = TestResult.__str__(self)
+    if self.content:
+      ret += " "+self.content+"\n"
+    if self.content_old:
+      ret += " "+self.content_old+"\n"
+    if self.content_exit:
+      ret += " "+self.content_exit+"\n"
+    return ret
+
 class HtmlTestResult(TestResult):
   ''' Represents the result of a http test '''
   def __init__(self, exit_node, website, status, reason=None, 
                content=None, content_exit=None, content_old=None):
-    super(HtmlTestResult, self).__init__(exit_node, website, status)
+    super(HtmlTestResult, self).__init__(exit_node, website, status, reason)
     self.proto = "http"
-    self.reason = reason
     self.content = content
     self.content_exit = content_exit
     self.content_old = content_old
-    self.false_positive=False
-    self.false_positive_reason="None"
 
   def mark_false_positive(self, reason):
     self.false_positive=True
@@ -202,6 +231,17 @@
     try: os.unlink(self.content_exit)
     except: pass
 
+  def __str__(self):
+    # XXX: Re-run the SoupDiffer+JSDiffer and compare these differences
+    ret = TestResult.__str__(self)
+    if self.content:
+      ret += " "+self.content+"\n"
+    if self.content_old:
+      ret += " "+self.content_old+"\n"
+    if self.content_exit:
+      ret += " "+self.content_exit+"\n"
+    return ret
+
 class SSHTestResult(TestResult):
   ''' Represents the result of an ssh test '''
   def __init__(self, exit_node, ssh_site, status):
@@ -382,7 +422,7 @@
     """ Create a map of changed tags to ALL attributes that tag
         has ever had (changed or not) """
     changed_tags = {}
-    for tags in map(BeautifulSoup, self.changed_tags()):
+    for tags in map(TheChosenSoup, self.changed_tags()):
       for t in tags.findAll():
         if t.name not in changed_tags:
           changed_tags[t.name] = sets.Set([])
@@ -394,7 +434,7 @@
     """ Returns true if we have additional tags with additional
         attributes that were not present in tag_attr_map 
         (returned from changed_tags_with_attrs) """
-    for tags in map(BeautifulSoup, self.changed_tags()):
+    for tags in map(TheChosenSoup, self.changed_tags()):
       for t in tags.findAll():
         if t.name not in tag_attr_map:
           return True

Added: torflow/trunk/NetworkScanners/snakeinspector.py
===================================================================
--- torflow/trunk/NetworkScanners/snakeinspector.py	                        (rev 0)
+++ torflow/trunk/NetworkScanners/snakeinspector.py	2009-02-09 10:36:42 UTC (rev 18443)
@@ -0,0 +1,39 @@
+#!/usr/bin/python
+
+import dircache
+import operator
+import os
+import pickle
+import sys
+import time
+
+import sets
+from sets import Set
+
+import libsoat
+from libsoat import *
+
+sys.path.append("../")
+from TorCtl.TorUtil import *
+
+
+def main(argv):
+  dh = DataHandler()
+  # FIXME: Handle this better.. maybe explicit --file or --exit options?
+  # For now, I should be the only one runnin this so...
+  # XXX: Also want to filter on reason, false positive, and
+  # failure/inconclusive
+  if len(argv) == 1:
+    results = dh.getAll()
+  elif argv[1][0] == '$':
+    results = dh.filterByNode(dh.getAll(), argv[1])
+  else:
+    results = [dh.getResult(argv[1])]
+
+  for r in results:
+    if r.status == TEST_FAILURE:
+      print r
+      print "\n-----------------------------\n"
+
+if __name__ == "__main__":
+  main(sys.argv)


Property changes on: torflow/trunk/NetworkScanners/snakeinspector.py
___________________________________________________________________
Added: svn:executable
   + *

Modified: torflow/trunk/NetworkScanners/soat.py
===================================================================
--- torflow/trunk/NetworkScanners/soat.py	2009-02-09 09:56:43 UTC (rev 18442)
+++ torflow/trunk/NetworkScanners/soat.py	2009-02-09 10:36:42 UTC (rev 18443)
@@ -59,9 +59,9 @@
 import OpenSSL
 from OpenSSL import *
 
+
 sys.path.append("./libs/")
-# XXX: Try to determine if we should be using MinimalSoup
-from BeautifulSoup.BeautifulSoup import BeautifulSoup, SoupStrainer, Tag
+from BeautifulSoup.BeautifulSoup import SoupStrainer, Tag
 from SocksiPy import socks
 import Pyssh.pyssh
 
@@ -286,7 +286,7 @@
   
         links = SoupStrainer('a')
         try:
-          soup = BeautifulSoup(content, parseOnlyThese=links)
+          soup = TheChosenSoup(content, parseOnlyThese=links)
         except Exception:
           plog('ERROR', 'Soup-scraping of http://'+host+search_path+" failed")
           traceback.print_exc()
@@ -875,17 +875,17 @@
     elements = SoupStrainer(lambda name, attrs: name in tags_to_check or 
      len(Set(map(lambda a: a[0], attrs)).intersection(Set(attrs_to_check))) > 0)
 
-    orig_soup = self._recursive_strain(BeautifulSoup(orig_html.decode('ascii',
+    orig_soup = self._recursive_strain(TheChosenSoup(orig_html.decode('ascii',
                                        'ignore'), parseOnlyThese=elements))
 
-    tor_soup = self._recursive_strain(BeautifulSoup(tor_html.decode('ascii',
+    tor_soup = self._recursive_strain(TheChosenSoup(tor_html.decode('ascii',
                                       'ignore'), parseOnlyThese=elements))
 
     # Also find recursive urls
     recurse_elements = SoupStrainer(lambda name, attrs: 
         name in tags_to_recurse and 
        len(Set(map(lambda a: a[0], attrs)).intersection(Set(attrs_to_recurse))) > 0)
-    self._add_recursive_targets(BeautifulSoup(tor_html.decode('ascii',
+    self._add_recursive_targets(TheChosenSoup(tor_html.decode('ascii',
                                    'ignore'), recurse_elements), address) 
 
     # compare the content
@@ -909,7 +909,7 @@
       return TEST_INCONCLUSIVE
 
 
-    new_soup = self._recursive_strain(BeautifulSoup(content_new,
+    new_soup = self._recursive_strain(TheChosenSoup(content_new,
                                      parseOnlyThese=elements))
     # compare the new and old content
     # if they match, means the node has been changing the content
@@ -1034,6 +1034,16 @@
     # return the cert
     return c.get_peer_certificate()
 
+  def get_resolved_ip(self, hostname):
+    mappings = self.mt.__control.get_address_mappings("cache")
+    ret = None
+    for m in mappings:
+      if m.from_name == hostname:
+        if ret:
+          plog("WARN", "Multiple maps for "+hostname)
+        ret = m.to_name
+    return ret
+
   def check_openssl(self, address):
     ''' check whether an https connection to a given address is molested '''
     plog('INFO', 'Conducting an ssl test with destination ' + address)