[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r20318: {torflow} Add ability to display only results that survived a network- (in torflow/trunk/NetworkScanners/ExitAuthority: . data/soat/dns data/soat/dnsrebind data/soat/http data/soat/imap data/soat/pop data/soat/smtp data/soat/ssh data/soat/ssl)



Author: mikeperry
Date: 2009-08-16 01:16:04 -0400 (Sun, 16 Aug 2009)
New Revision: 20318

Added:
   torflow/trunk/NetworkScanners/ExitAuthority/data/soat/dns/confirmed/
   torflow/trunk/NetworkScanners/ExitAuthority/data/soat/dnsrebind/confirmed/
   torflow/trunk/NetworkScanners/ExitAuthority/data/soat/http/confirmed/
   torflow/trunk/NetworkScanners/ExitAuthority/data/soat/imap/confirmed/
   torflow/trunk/NetworkScanners/ExitAuthority/data/soat/pop/confirmed/
   torflow/trunk/NetworkScanners/ExitAuthority/data/soat/smtp/confirmed/
   torflow/trunk/NetworkScanners/ExitAuthority/data/soat/ssh/confirmed/
   torflow/trunk/NetworkScanners/ExitAuthority/data/soat/ssl/confirmed/
Modified:
   torflow/trunk/NetworkScanners/ExitAuthority/libsoat.py
   torflow/trunk/NetworkScanners/ExitAuthority/snakeinspector.py
   torflow/trunk/NetworkScanners/ExitAuthority/soat.py
Log:

Add ability to display only results that survived a
network-wide filtering round (we'll call those "confirmed",
which is different and possibly stronger than "rescanned", due
to the ordering of our filter application as it is now).



Modified: torflow/trunk/NetworkScanners/ExitAuthority/libsoat.py
===================================================================
--- torflow/trunk/NetworkScanners/ExitAuthority/libsoat.py	2009-08-16 02:15:23 UTC (rev 20317)
+++ torflow/trunk/NetworkScanners/ExitAuthority/libsoat.py	2009-08-16 05:16:04 UTC (rev 20318)
@@ -109,11 +109,12 @@
     self.reason = reason
     self.extra_info = None
     self.false_positive=False
+    self.confirmed=False
     self.false_positive_reason="None"
     self.verbose=0
     self.from_rescan = False
     self.filename=None
-    self._pickle_revision = 5
+    self._pickle_revision = 6
 
   def depickle_upgrade(self):
     if not "_pickle_revision" in self.__dict__: # upgrade to v0
@@ -133,8 +134,10 @@
     if self._pickle_revision < 5:
       self._pickle_revision = 5
       if type(self.exit_ip) == str or not self.exit_ip: self.exit_ip = 0
+    if self._pickle_revision < 6:
+      self._pickle_revision = 6
+      self.confirmed=False
 
-
   def _rebase(self, filename, new_data_root):
     if not filename: return filename
     filename = os.path.normpath(filename)
@@ -176,6 +179,8 @@
       ret += "\n Removed as False Positive: "+self.false_positive_reason
     if self.from_rescan:
       ret += "\n From rescan: "+str(self.from_rescan)
+    if self.confirmed:
+      ret += "\n Confirmed. "
     ret += "\n"
     return ret
 
@@ -629,7 +634,7 @@
     return str(safe_file[:200])
   safeFilename = Callable(safeFilename)
 
-  def __resultFilename(self, result):
+  def __resultFilename(self, result, confirmed=False):
     address = ''
     if result.__class__.__name__ == 'HtmlTestResult' or result.__class__.__name__ == 'HttpTestResult':
       address = DataHandler.safeFilename(result.site[7:])
@@ -641,7 +646,9 @@
       raise Exception, 'This doesn\'t seems to be a result instance.'
 
     rdir = self.data_dir+result.proto.lower()+'/'
-    if result.false_positive:
+    if confirmed:
+      rdir += 'confirmed/'
+    elif result.false_positive:
       rdir += 'falsepositive/'
     elif result.from_rescan:
       rdir += 'rescan/'
@@ -654,9 +661,9 @@
 
     return DataHandler.uniqueFilename(str((rdir+address+'.'+result.exit_node[1:]+".result").decode('ascii', 'ignore')))
 
-  def saveResult(self, result):
+  def saveResult(self, result, confirmed=False):
     ''' generic method for saving test results '''
-    result.filename = self.__resultFilename(result)
+    result.filename = self.__resultFilename(result, confirmed)
     SnakePickler.dump(result, result.filename)
 
   def __testFilename(self, test, position=-1):

Modified: torflow/trunk/NetworkScanners/ExitAuthority/snakeinspector.py
===================================================================
--- torflow/trunk/NetworkScanners/ExitAuthority/snakeinspector.py	2009-08-16 02:15:23 UTC (rev 20317)
+++ torflow/trunk/NetworkScanners/ExitAuthority/snakeinspector.py	2009-08-16 05:16:04 UTC (rev 20318)
@@ -53,10 +53,10 @@
 
 def getargs(argv):
   try:
-    opts,args = getopt.getopt(argv[1:],"d:f:e:r:vt:p:s:o:n:a:b:Fm", 
-             ["dir=", "file=", "exit=", "reason=", "resultfilter=", "proto=", 
+    opts,args = getopt.getopt(argv[1:],"d:f:e:r:vt:p:s:o:n:a:b:Fmc",
+             ["dir=", "file=", "exit=", "reason=", "resultfilter=", "proto=",
               "verbose", "statuscode=", "sortby=", "noreason=", "after=",
-              "before=", "falsepositives", "email"])
+              "before=", "falsepositives", "email", "confirmed"])
   except getopt.GetoptError,err:
     print str(err)
     usage(argv)
@@ -75,6 +75,7 @@
   sortby="proto"
   falsepositives=False
   send_email = False
+  confirmed = False
   for o,a in opts:
     if o == '-d' or o == '--dir':
       use_dir = a
@@ -98,6 +99,8 @@
       proto = a
     elif o == '-F' or o == '--falsepositives':
       falsepositives = True
+    elif o == '-c' or o == '--confirmed':
+      confirmed = True
     elif o == '-s' or o == '--sortby': 
       if a not in ["proto", "site", "exit", "reason"]:
         usage(argv)
@@ -107,7 +110,7 @@
         result = int(a)
       except ValueError:
         result = RESULT_CODES[a]
-  return use_dir,use_file,node,reasons,noreasons,result,verbose,resultfilter,proto,sortby,before,after,falsepositives,send_email
+  return use_dir,use_file,node,reasons,noreasons,result,verbose,resultfilter,proto,sortby,before,after,falsepositives,send_email,confirmed
 
 def send_mail(fro, to, subject, text, server, files=[]):
   assert type(to)==list
@@ -136,7 +139,7 @@
 
 def main(argv):
   now = time.time()
-  use_dir,use_file,node,reasons,noreasons,result,verbose,resultfilter,proto,sortby,before,after,falsepositives,send_email=getargs(argv)
+  use_dir,use_file,node,reasons,noreasons,result,verbose,resultfilter,proto,sortby,before,after,falsepositives,send_email,confirmed=getargs(argv)
   dh = DataHandler(use_dir)
 
   if use_file:
@@ -161,6 +164,7 @@
     if reasons and r.reason not in reasons: continue
     if r.timestamp < after or before < r.timestamp: continue
     if (falsepositives) ^ r.false_positive: continue
+    if confirmed != r.confirmed: continue
     if (not result or r.status == result) and \
        (not proto or r.proto == proto) and \
        (not resultfilter or r.__class__.__name__ == resultfilter):

Modified: torflow/trunk/NetworkScanners/ExitAuthority/soat.py
===================================================================
--- torflow/trunk/NetworkScanners/ExitAuthority/soat.py	2009-08-16 02:15:23 UTC (rev 20317)
+++ torflow/trunk/NetworkScanners/ExitAuthority/soat.py	2009-08-16 05:16:04 UTC (rev 20318)
@@ -489,8 +489,9 @@
         self.remove_target(address, failtype)
 
   def remove_false_positives(self):
-    if self.rescan_nodes: 
+    if self.rescan_nodes:
       plog("INFO", "Not removing false positives for rescan of "+self.__class__.__name__)
+      return
     else:
       plog("INFO", "Removing false positives for "+self.__class__.__name__)
     self._remove_false_positive_type(self.exit_fails,
@@ -502,6 +503,10 @@
     self._remove_false_positive_type(self.connect_fails,
                                      FALSEPOSITIVE_DEADSITE,
                                      max_connect_fail_pct)
+    for r in self.results:
+      if not r.false_positive and r.status == TEST_FAILURE:
+        r.confirmed=True
+        datahandler.saveResult(r, confirmed=True)
 
   def _reset(self):
     self.results = []