[or-cvs] r18387: {torflow} Toss excessively dynamic pages into a special results dir fo (in torflow/trunk/NetworkScanners: . data/soat/http)

[mikeperry@xxxxxxxx]

Author: mikeperry
Date: 2009-02-04 09:02:46 -0500 (Wed, 04 Feb 2009)
New Revision: 18387

Added:
   torflow/trunk/NetworkScanners/data/soat/http/falsepositive/
Modified:
   torflow/trunk/NetworkScanners/README.ExitScanning
   torflow/trunk/NetworkScanners/libsoat.py
   torflow/trunk/NetworkScanners/soat.py
Log:

Toss excessively dynamic pages into a special results dir for
false positives to make it easier to process results from
other people's SoaT runs.



Modified: torflow/trunk/NetworkScanners/README.ExitScanning
===================================================================
--- torflow/trunk/NetworkScanners/README.ExitScanning	2009-02-04 13:56:48 UTC (rev 18386)
+++ torflow/trunk/NetworkScanners/README.ExitScanning	2009-02-04 14:02:46 UTC (rev 18387)
@@ -75,8 +75,8 @@
 languages (especially any that have unicode characters), using keywords
 from them would be especially useful for testing and scanning. Note
 that these queries WILL be issued in plaintext via non-Tor, and the
-resulting urls fetched via non-Tor as well, so bear that in mind for
-your legal jurisdiction when choosing keywords.
+resulting urls fetched via non-Tor as well, so bear that and your
+server's legal jurisdiction in mind when choosing keywords.
 
 You can also separate out the wordlist.txt file into three files by
 changing the soat_config.py settings 'filetype_wordlist_file',

Modified: torflow/trunk/NetworkScanners/libsoat.py
===================================================================
--- torflow/trunk/NetworkScanners/libsoat.py	2009-02-04 13:56:48 UTC (rev 18386)
+++ torflow/trunk/NetworkScanners/libsoat.py	2009-02-04 14:02:46 UTC (rev 18387)
@@ -55,7 +55,22 @@
     self.site = site
     self.timestamp = time.time()
     self.status = status
+    self.false_positive=False
+  
+  def mark_false_positive(self):
+    pass
 
+  def move_file(self, file, to_dir):
+    try:
+      basename = os.path.basename(file)
+      new_file = to_dir+basename
+      os.rename(file, new_file)
+      return new_file
+    except:
+      traceback.print_exc()
+      plog("WARN", "Error moving "+file+" to "+dir)
+      return file
+
 class SSLTestResult(TestResult):
   ''' Represents the result of an openssl test '''
   def __init__(self, exit_node, ssl_site, cert_file, status):
@@ -78,7 +93,12 @@
     self.content_exit = content_exit
     self.content_old = content_old
 
-  # XXX: Instead of removing these, move them to a 'falsepositives' dir
+  def mark_false_positive(self):
+    self.false_positive=True
+    self.content=self.move_file(self.content, http_falsepositive_dir)
+    self.content_old=self.move_file(self.content_old, http_falsepositive_dir)
+    self.content_exit=self.move_file(self.content_exit,http_falsepositive_dir)
+
   def remove_files(self):
     try: os.unlink(self.content)
     except: pass
@@ -107,6 +127,12 @@
     self.content_exit = content_exit
     self.content_old = content_old
 
+  def mark_false_positive(self):
+    self.false_positive=True
+    self.content=self.move_file(self.content, http_falsepositive_dir)
+    self.content_old=self.move_file(self.content_old, http_falsepositive_dir)
+    self.content_exit=self.move_file(self.content_exit,http_falsepositive_dir)
+
   def remove_files(self):
     try: os.unlink(self.content)
     except: pass
@@ -130,6 +156,15 @@
     self.content_exit = content_exit
     self.content_old = content_old
 
+  def mark_false_positive(self):
+    self.false_positive=True
+    self.tags=self.move_file(self.tags,http_falsepositive_dir)
+    self.tags_old=self.move_file(self.tags_old,http_falsepositive_dir)
+    self.exit_tags=self.move_file(self.exit_tags,http_falsepositive_dir)
+    self.content=self.move_file(self.content,http_falsepositive_dir)
+    self.content_old=self.move_file(self.content_old, http_falsepositive_dir)
+    self.content_exit=self.move_file(self.content_exit,http_falsepositive_dir)
+
   def remove_files(self):
     try: os.unlink(self.tags)
     except: pass

Modified: torflow/trunk/NetworkScanners/soat.py
===================================================================
--- torflow/trunk/NetworkScanners/soat.py	2009-02-04 13:56:48 UTC (rev 18386)
+++ torflow/trunk/NetworkScanners/soat.py	2009-02-04 14:02:46 UTC (rev 18387)
@@ -307,7 +307,8 @@
     self.exit_fails = {}
     self.successes = {}
     self.exit_limit = 100
-    self.httpcode_limit = 100
+    # XXX: 3 is waaay too low. 100 is more like it.. But set for testing
+    self.httpcode_limit = 3
     self.scan_filetypes = filetypes
     self.results = []
 
@@ -379,8 +380,7 @@
       if r.site == address:
         kill_results.append(r)
     for r in kill_results:
-      # XXX: Move files instead of removing them..
-      #r.remove_files()
+      r.mark_false_positive()
       self.results.remove(r)
     
   def register_exit_failure(self, address, exit_node):
@@ -632,7 +632,9 @@
     self.recurse_filetypes = recurse_filetypes
     self.fetch_queue = Queue.Queue()
     self.dynamic_fails = {}
-    self.dynamic_limit = 10
+    # XXX: 3 is way too low, but set for code exercise. 10 
+    # is prob reasonable
+    self.dynamic_limit = 3
  
   def run_test(self):
     # A single test should have a single cookie jar