[or-cvs] r18515: {torflow} Make false positive filter log the differences it sees at th (torflow/trunk/NetworkScanners)

[mikeperry@xxxxxxxx]

Author: mikeperry
Date: 2009-02-12 22:22:26 -0500 (Thu, 12 Feb 2009)
New Revision: 18515

Modified:
   torflow/trunk/NetworkScanners/libsoat.py
   torflow/trunk/NetworkScanners/snakeinspector.py
   torflow/trunk/NetworkScanners/soat.py
Log:

Make false positive filter log the differences it sees at the
time, because Non-Tor content gets updated on disk as it
changes. Also add sorting options to snakeinspector.py.



Modified: torflow/trunk/NetworkScanners/libsoat.py
===================================================================
--- torflow/trunk/NetworkScanners/libsoat.py	2009-02-13 00:50:45 UTC (rev 18514)
+++ torflow/trunk/NetworkScanners/libsoat.py	2009-02-13 03:22:26 UTC (rev 18515)
@@ -112,10 +112,10 @@
     ret += " "+str(RESULT_STRINGS[self.status])
     if self.reason:
       ret += " Reason: "+self.reason
-    if self.extra_info:
-      ret += " "+self.extra_info 
     if self.false_positive:
       ret += " (False positive: "+self.false_positive_reason+")"
+    if self.extra_info:
+      ret += "\n Extra info: "+self.extra_info 
     if self.from_rescan:
       ret += "\nFrom rescan: "+str(self.from_rescan)
     ret += "\n"

Modified: torflow/trunk/NetworkScanners/snakeinspector.py
===================================================================
--- torflow/trunk/NetworkScanners/snakeinspector.py	2009-02-13 00:50:45 UTC (rev 18514)
+++ torflow/trunk/NetworkScanners/snakeinspector.py	2009-02-13 03:22:26 UTC (rev 18515)
@@ -31,14 +31,15 @@
   print "  --proto <protocol>"
   print "  --resultfilter <TestResult class name>"
   print "  --statuscode <'Failure' or 'Inconclusive'>"
+  print "  --sortby <'proto' or 'url' or 'exit' or 'reason'>"
   print "  --verbose"
   sys.exit(1)
 
 def getargs(argv):
   try:
-    opts,args = getopt.getopt(argv[1:],"d:f:e:r:vt:p:s:", 
+    opts,args = getopt.getopt(argv[1:],"d:f:e:r:vt:p:s:o:", 
              ["dir=", "file=", "exit=", "reason=", "resultfilter=", "proto=", 
-              "verbose", "statuscode="])
+              "verbose", "statuscode=", "sortby="])
   except getopt.GetoptError,err:
     print str(err)
     usage(argv)
@@ -50,6 +51,7 @@
   verbose=1
   proto=None
   resultfilter=None
+  sortby="proto"
   for o,a in opts:
     if o == '-d' or o == '--dir':
       use_dir = a
@@ -65,15 +67,19 @@
       resultfilter = a
     elif o == '-p' or o == '--proto':
       proto = a
+    elif o == '-s' or o == '--sortby': 
+      if a not in ["proto", "site", "exit", "reason"]:
+        usage(argv)
+      else: sortby = a 
     elif o == '-s' or o == '--statuscode': 
       try:
         result = int(a)
       except ValueError:
         result = RESULT_CODES[a]
-  return use_dir,use_file,node,reason,result,verbose,resultfilter,proto
+  return use_dir,use_file,node,reason,result,verbose,resultfilter,proto,sortby
  
 def main(argv):
-  use_dir,use_file,node,reason,result,verbose,resultfilter,proto=getargs(argv)
+  use_dir,use_file,node,reason,result,verbose,resultfilter,proto,sortby=getargs(argv)
   dh = DataHandler(use_dir)
   print dh.data_dir
 
@@ -84,6 +90,13 @@
   else:
     results = dh.getAll()
 
+  if sortby == "url":
+    results.sort(lambda x, y: cmp(x.site, y.site))
+  elif sortby == "reason":
+    results.sort(lambda x, y: cmp(x.reason, y.reason))
+  elif sortby == "exit":
+    results.sort(lambda x, y: cmp(x.exit_node, y.exit_node))
+
   for r in results:
     r.verbose = verbose
     if (not result or r.status == result) and \

Modified: torflow/trunk/NetworkScanners/soat.py
===================================================================
--- torflow/trunk/NetworkScanners/soat.py	2009-02-13 00:50:45 UTC (rev 18514)
+++ torflow/trunk/NetworkScanners/soat.py	2009-02-13 03:22:26 UTC (rev 18515)
@@ -1074,22 +1074,31 @@
                             new_vs_old.changed_attributes_by_tag())
 
     changed_content = bool(new_vs_old.changed_content() or old_vs_new.changed_content())
+
+    more_tags = new_vs_tor.more_changed_tags(changed_tags)     
+    more_attrs = new_vs_tor.more_changed_attrs(changed_attributes)
+    more_content = new_vs_tor.changed_content()
  
     # Verify all of our changed tags are present here 
-    if new_vs_tor.has_more_changed_tags(changed_tags) or \
-      new_vs_tor.has_more_changed_attrs(changed_attributes) or \
-      new_vs_tor.changed_content() and not changed_content:
+    if more_tags or more_attrs or (more_content and not changed_content):
       false_positive = False
+      plog("NOTICE", "SoupDiffer finds differences for "+address)
+      plog("NOTICE", "New Tags:\n"+more_tags)
+      plog("NOTICE", "New Attrs:\n"+more_attrs)
+      if more_content and not changed_content:
+        plog("NOTICE", "New Content:\n"+more_content)
     else:
+      plog("INFO", "SoupDiffer predicts false_positive")
       false_positive = True
-      
-    plog("INFO", "SoupDiffer predicts false_positive="+str(false_positive))
 
     if false_positive:
       jsdiff = JSSoupDiffer(orig_soup)
       jsdiff.prune_differences(new_soup)
-      false_positive = not jsdiff.contains_differences(tor_soup)
+      differences = jsdiff.show_differences(tor_soup)
+      false_positive = not differences
       plog("INFO", "JSSoupDiffer predicts false_positive="+str(false_positive))
+      if not false_positive:
+        plog("NOTICE", "JSSoupDiffer finds differences: "+differences)
 
     if false_positive:
       plog("NOTICE", "False positive detected for dynamic change at "+address+" via "+exit_node)
@@ -2353,6 +2362,7 @@
   tests = {}
 
   if do_resume:
+    plog("NOTICE", "Resuming previous SoaT run")
     if do_ssl:
       tests["SSL"] = datahandler.loadTest("SSLTest", resume_run)
 
@@ -2378,6 +2388,7 @@
     sys.exit(0)
 
   if do_rescan:
+    plog("NOTICE", "Loading rescan.")
     for test in tests.itervalues():
       test.load_rescan(TEST_FAILURE)