[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r18223: {torflow} Factor out soatstats.py common code into libsoat.py. Fix som (torflow/trunk/NetworkScanners)



Author: mikeperry
Date: 2009-01-22 07:16:05 -0500 (Thu, 22 Jan 2009)
New Revision: 18223

Added:
   torflow/trunk/NetworkScanners/libsoat.py
Modified:
   torflow/trunk/NetworkScanners/soat.py
Log:

Factor out soatstats.py common code into libsoat.py. Fix some
log levels in soat so that it only gives us ERROR's if either
the Tor network is on fire or soat is. Also fix a couple
pychecker warns.




Added: torflow/trunk/NetworkScanners/libsoat.py
===================================================================
--- torflow/trunk/NetworkScanners/libsoat.py	                        (rev 0)
+++ torflow/trunk/NetworkScanners/libsoat.py	2009-01-22 12:16:05 UTC (rev 18223)
@@ -0,0 +1,197 @@
+#!/usr/bin/python
+#
+# Common code to soat
+
+import dircache
+import operator
+import os
+import pickle
+import sys
+import time
+
+import sets
+from sets import Set
+
+#
+# Data storage
+#
+
+# data locations
+
+data_dir = './data/soat/'
+ssl_certs_dir = data_dir + 'ssl/certs/'
+http_tags_dir = data_dir + 'http/tags/'
+
+# constants
+
+TEST_SUCCESS = 0
+TEST_INCONCLUSIVE = 1
+TEST_FAILURE = 2
+
+# classes to use with pickle to dump test results into files
+
+class TestResult(object):
+    ''' Parent class for all test result classes '''
+    def __init__(self, exit_node, site, status):
+        self.exit_node = exit_node
+        self.site = site
+        self.timestamp = time.time()
+        self.status = status
+
+class SSLTestResult(TestResult):
+    ''' Represents the result of an openssl test '''
+    def __init__(self, exit_node, ssl_site, cert_file, status):
+        super(SSLTestResult, self).__init__(exit_node, ssl_site, status)
+        self.cert = cert_file
+
+class HttpTestResult(TestResult):
+    ''' Represents the result of a http test '''
+    def __init__(self, exit_node, website, tag_prints, status):
+        super(HttpTestResult, self).__init__(exit_node, website, status)
+        self.tag_prints = tag_prints
+
+class SSHTestResult(TestResult):
+    ''' Represents the result of an ssh test '''
+    def __init__(self, exit_node, ssh_site, status):
+        super(SSHTestResult, self).__init__(exit_node, ssh_site, status)
+
+class DNSTestResult(TestResult):
+    ''' Represents the result of a dns test '''
+    def __init__(self, exit_node, dns_site, status):
+        super(DNSTestResult, self).__init__(exit_node, dns_site, status)
+
+class DNSRebindTestResult(TestResult):
+    ''' Represents the result of a dns rebind test '''
+    def __init__(self, exit_node, dns_rebind_site, status):
+        super(DNSRebindTestResult, self).__init__(exit_node, dns_rebind_site, status)
+
+class SMTPTestResult(TestResult):
+    ''' Represents the result of an smtp test '''
+    def __init__(self, exit_node, smtp_site, status):
+        super(SMTPTestResult, self).__init__(exit_node, smtp_site, status)
+
+class IMAPTestResult(TestResult):
+    ''' Represents the result of an imap test '''
+    def __init__(self, exit_node, imap_site, status):
+        super(IMAPTestResult, self).__init__(exit_node, imap_site, status)
+
+class POPTestResult(TestResult):
+    ''' Represents the result of a pop test '''
+    def __init__(self, exit_node, pop_site, status):
+        super(POPTestResult, self).__init__(exit_node, pop_site, status)
+
+class DataHandler:
+    ''' Class for saving and managing test result data '''
+    def filterResults(self, results, protocols=[], show_good=False, 
+            show_bad=False, show_inconclusive=False):
+        ''' filter results based on protocol and success level ''' 
+
+        protocol_filters = []
+        status_filters = []
+
+        for protocol in protocols:
+            protocol_filters.append(lambda x, p=protocol: x.__class__.__name__.lower()[:-10].endswith(p))
+        if show_good:
+            status_filters.append(lambda x: x.status == TEST_SUCCESS)
+        if show_bad:
+            status_filters.append(lambda x: x.status == TEST_FAILURE)
+        if show_inconclusive:
+            status_filters.append(lambda x: x.status == TEST_INCONCLUSIVE)
+
+        if len(protocol_filters) == 0 or len(status_filters) == 0:
+            return []
+       
+        protocol_filter = lambda x: reduce(operator.__or__, [f(x) for f in protocol_filters])
+        status_filter = lambda x: reduce(operator.__or__, [f(x) for f in status_filters])
+
+        return [x for x in results if (protocol_filter(x) and status_filter(x))]
+        
+    def filterByNode(self, results, id):
+        ''' filter by node'''
+        return filter(lambda x: x.exit_node == id, results)
+
+    def getAll(self):
+        ''' get all available results'''
+        return self.__getResults(data_dir)
+
+    def getSsh(self):
+        ''' get results of ssh tests '''
+        return self.__getResults(data_dir + 'ssh/')
+        
+    def getHttp(self):
+        ''' get results of http tests '''
+        return self.__getResults(data_dir + 'http/')
+
+    def getSsl(self):
+        ''' get results of ssl tests '''
+        return self.__getResults(data_dir + 'ssl/')
+
+    def getSmtp(self):
+        ''' get results of smtp tests '''
+        return self.__getResults(data_dir + 'smtp/')
+
+    def getPop(self):
+        ''' get results of pop tests '''
+        return self.__getResults(data_dir + 'pop/')
+
+    def getImap(self):
+        ''' get results of imap tests '''
+        return self.__getResults(data_dir + 'imap/')
+
+    def getDns(self):
+        ''' get results of basic dns tests '''
+        return self.__getResults(data_dir + 'dns')
+
+    def getDnsRebind(self):
+        ''' get results of dns rebind tests '''
+        return self.__getResults(data_dir + 'dnsbrebind/')
+
+    def __getResults(self, dir):
+        ''' 
+        recursively traverse the directory tree starting with dir
+        gather test results from files ending with .result
+        '''
+        results = []
+
+        for root, dirs, files in os.walk(dir):
+            for file in files:
+                if file.endswith('result'):
+                    fh = open(os.path.join(root, file))
+                    result = pickle.load(fh)
+                    results.append(result)
+
+        return results
+
+    def safeFilename(self, str):
+        ''' 
+        remove characters illegal in some systems 
+        and trim the string to a reasonable length
+        '''
+        replaced = (str.replace('/','_').replace('\\','_').replace('?','_').replace(':','_').
+            replace('|','_').replace('*','_').replace('<','_').replace('>','_').replace('"',''))
+        return replaced[:200]
+
+    def saveResult(self, result):
+        ''' generic method for saving test results '''
+        address = ''
+        if result.__class__.__name__ == 'HttpTestResult':
+            address = self.safeFilename(result.site[7:])
+        elif result.__class__.__name__ == 'SSLTestResult':
+            address = self.safeFilename(result.site[8:])
+        elif 'TestResult' in result.__class__.__name__:
+            address = self.safeFilename(result.site)
+        else:
+            raise Exception, 'This doesn\'t seems to be a result instance.'
+
+        dir = data_dir + result.__class__.__name__[:-10].lower() + '/'
+        if result.status == TEST_SUCCESS:
+            dir += 'successful/'
+        if result.status == TEST_INCONCLUSIVE:
+            dir += 'inconclusive/'
+        if result.status == TEST_FAILURE:
+            dir += 'failed/'
+        
+        result_file = open(dir + result.exit_node[1:] + "-" + address + '.result', 'w')
+        pickle.dump(result, result_file)
+        result_file.close()
+

Modified: torflow/trunk/NetworkScanners/soat.py
===================================================================
--- torflow/trunk/NetworkScanners/soat.py	2009-01-22 11:52:52 UTC (rev 18222)
+++ torflow/trunk/NetworkScanners/soat.py	2009-01-22 12:16:05 UTC (rev 18223)
@@ -39,8 +39,8 @@
 import urllib2
 import traceback
 
-import soatstats
-from soatstats import *
+import libsoat 
+from libsoat import *
 
 sys.path.append("../")
 
@@ -50,6 +50,7 @@
 from TorCtl.PathSupport import *
 from TorCtl.TorCtl import Connection, EventHandler
 
+import OpenSSL
 from OpenSSL import *
 
 sys.path.append("./libs/")
@@ -181,7 +182,7 @@
             reply = self.__meta.readline()
             if reply[:3] != '250': # first three chars indicate the reply code
                 reply += self.__meta.readline()
-                plog('ERROR', 'Error configuring metatroller (' + command + ' failed)')
+                plog('ERROR', 'Error configuring metatroller (' + c + ' failed)')
                 plog('ERROR', reply)
                 exit()
 
@@ -457,19 +458,19 @@
             plog('INFO', 'Opening a direct ssl connection to ' + address)
             original_cert = self.ssl_request(address)
             if not original_cert:
-                plog('ERROR', 'Error getting the correct cert for ' + address)
+                plog('WARN', 'Error getting the correct cert for ' + address)
                 return TEST_INCONCLUSIVE
             if original_cert.has_expired():
-                plog('ERROR', 'The ssl cert for ' + address + 'seems to have expired. Skipping to the next test...')
+                plog('WARN', 'The ssl cert for ' + address + 'seems to have expired. Skipping to the next test...')
                 return TEST_INCONCLUSIVE
             cert_file = open(ssl_certs_dir + address_file + '.pem', 'w')
             cert_file.write(crypto.dump_certificate(crypto.FILETYPE_PEM, original_cert))
             cert_file.close()
         except OpenSSL.crypto.Error:
-            plog('ERROR', 'There are non-related files in ' + ssl_certs_dir + '. You should probably clean it.')
+            plog('NOTICE', 'There are non-related files in ' + ssl_certs_dir + '. You should probably clean it.')
             return TEST_INCONCLUSIVE
         if not original_cert:
-            plog('ERROR', 'Error getting the correct cert for ' + address)
+            plog('WARN', 'Error getting the correct cert for ' + address)
             return TEST_INCONCLUSIVE
 
         # get an easily comparable representation of the certs
@@ -490,7 +491,7 @@
         plog('INFO', 'Opening a direct ssl connection to ' + address)
         original_cert_new = self.ssl_request(address)
         if original_cert_new == 0:
-            plog('ERROR', 'Error getting the correct cert for ' + address)
+            plog('WARN', 'Error getting the correct cert for ' + address)
             result = SSLTestResult(exit_node, address, 0, TEST_INCONCLUSIVE)
             self.__datahandler.saveResult(result)
             return TEST_INCONCLUSIVE
@@ -571,13 +572,13 @@
             if ehlo2_reply != 250:
                 raise smtplib.SMTPException('Second ehlo failed')
         except socket.gaierror, e:
-            plog('ERROR', 'A connection error occured while testing smtp at ' + address)
-            plog('ERROR', e)
+            plog('WARN', 'A connection error occured while testing smtp at ' + address)
+            plog('WARN', e)
             socket.socket = defaultsocket
             return TEST_INCONCLUSIVE
         except smtplib.SMTPException, e:
-            plog('ERROR','An error occured while testing smtp at ' + address)
-            plog('ERROR', e)
+            plog('WARN','An error occured while testing smtp at ' + address)
+            plog('WARN', e)
             return TEST_INCONCLUSIVE
         # reset the connection method back to direct
         socket.socket = defaultsocket 
@@ -607,13 +608,13 @@
             if ehlo2_reply_d != 250:
                 raise smtplib.SMTPException('Second ehlo failed')
         except socket.gaierror, e:
-            plog('ERROR', 'A connection error occured while testing smtp at ' + address)
-            plog('ERROR', e)
+            plog('WARN', 'A connection error occured while testing smtp at ' + address)
+            plog('WARN', e)
             socket.socket = defaultsocket
             return TEST_INCONCLUSIVE
         except smtplib.SMTPException, e:
-            plog('ERROR', 'An error occurred while testing smtp at ' + address)
-            plog('ERROR', e)
+            plog('WARN', 'An error occurred while testing smtp at ' + address)
+            plog('WARN', e)
             return TEST_INCONCLUSIVE
 
         print ehlo1_reply, ehlo1_reply_d, has_starttls, has_starttls_d, ehlo2_reply, ehlo2_reply_d
@@ -673,6 +674,7 @@
             if starttls_present:
                 pop.writeline('STLS')
 
+            starttls_response = pop.readline()
             starttls_started = '+OK' in starttls_response
 
             # negotiate TLS and issue some request to feel good about it
@@ -702,13 +704,13 @@
                     tls_succeeded = False
 
         except socket.error, e: 
-            plog('ERROR', 'Connection to ' + address + ':' + port + ' refused')
-            plog('ERROR', e)
+            plog('WARN', 'Connection to ' + address + ':' + port + ' refused')
+            plog('WARN', e)
             socket.socket = defaultsocket
             return TEST_INCONCLUSIVE
         except OpenSSL.SSL.SysCallError, e:
-            plog('ERROR', 'Error while negotiating an SSL connection to ' + address + ':' + port)
-            plog('ERROR', e)
+            plog('WARN', 'Error while negotiating an SSL connection to ' + address + ':' + port)
+            plog('WARN', e)
             socket.socket = defaultsocket
             return TEST_INCONCLUSIVE
 
@@ -781,19 +783,19 @@
                     tls_succeeded_d = False
 
         except socket.error, e: 
-            plog('ERROR', 'Connection to ' + address + ':' + port + ' refused')
-            plog('ERROR', e)
+            plog('WARN', 'Connection to ' + address + ':' + port + ' refused')
+            plog('WARN', e)
             socket.socket = defaultsocket
             return TEST_INCONCLUSIVE
         except OpenSSL.SSL.SysCallError, e:
-            plog('ERROR', 'Error while negotiating an SSL connection to ' + address + ':' + port)
-            plog('ERROR', e)
+            plog('WARN', 'Error while negotiating an SSL connection to ' + address + ':' + port)
+            plog('WARN', e)
             socket.socket = defaultsocket
             return TEST_INCONCLUSIVE
 
         # compare
         if (capabilities_ok != capabilities_ok_d or starttls_present != starttls_present_d or 
-                tls_started != tls_started_d or tls_suceeded != tls_succeeded_d):
+                tls_started != tls_started_d or tls_succeeded != tls_succeeded_d):
             result = POPTestResult(exit_node, address, TEST_FAILURE)
             self.__datahandler.saveResult(result)
             return TEST_FAILURE
@@ -867,13 +869,13 @@
                     tls_succeeded = False
     
         except socket.error, e: 
-            plog('ERROR', 'Connection to ' + address + ':' + port + ' refused')
-            plog('ERROR', e)
+            plog('WARN', 'Connection to ' + address + ':' + port + ' refused')
+            plog('WARN', e)
             socket.socket = defaultsocket
             return TEST_INCONCLUSIVE
         except OpenSSL.SSL.SysCallError, e:
-            plog('ERROR', 'Error while negotiating an SSL connection to ' + address + ':' + port)
-            plog('ERROR', e)
+            plog('WARN', 'Error while negotiating an SSL connection to ' + address + ':' + port)
+            plog('WARN', e)
             socket.socket = defaultsocket
             return TEST_INCONCLUSIVE
         
@@ -936,13 +938,13 @@
                     tls_succeeded_d = False
 
         except socket.error, e: 
-            plog('ERROR', 'Connection to ' + address + ':' + port + ' refused')
-            plog('ERROR', e)
+            plog('WARN', 'Connection to ' + address + ':' + port + ' refused')
+            plog('WARN', e)
             socket.socket = defaultsocket
             return TEST_INCONCLUSIVE
         except OpenSSL.SSL.SysCallError, e:
-            plog('ERROR', 'Error while negotiating an SSL connection to ' + address + ':' + port)
-            plog('ERROR', e)
+            plog('WARN', 'Error while negotiating an SSL connection to ' + address + ':' + port)
+            plog('WARN', e)
             socket.socket = defaultsocket
             return TEST_INCONCLUSIVE
 
@@ -976,8 +978,8 @@
             for result in results:
                 ips_d.add(result[4][0])
         except socket.herror, e:
-            plog('ERROR', 'An error occured while performing a basic dns test')
-            plog('ERROR', e)
+            plog('WARN', 'An error occured while performing a basic dns test')
+            plog('WARN', e)
             return TEST_INCONCLUSIVE
 
         if ip in ips_d:
@@ -1008,15 +1010,15 @@
             reply = urllib2.urlopen(request)
             content = reply.read()
         except (ValueError, urllib2.URLError):
-            plog('ERROR', 'The http-request address ' + address + ' is malformed')
+            plog('WARN', 'The http-request address ' + address + ' is malformed')
             return 0
         except (IndexError, TypeError):
-            plog('ERROR', 'An error occured while negotiating socks5 with Tor')
+            plog('WARN', 'An error occured while negotiating socks5 with Tor')
             return 0
         except KeyboardInterrupt:
             raise KeyboardInterrupt
         except:
-            plog('ERROR', 'An unknown HTTP error occured')
+            plog('WARN', 'An unknown HTTP error occured')
             traceback.print_exc()
             return 0
 
@@ -1048,14 +1050,14 @@
             c.connect((address, 443))
             c.send(crypto.dump_certificate_request(crypto.FILETYPE_PEM,request))
         except socket.error, e:
-            plog('ERROR','An error occured while opening an ssl connection to ' + address)
-            plog('ERROR', e)
+            plog('WARN','An error occured while opening an ssl connection to ' + address)
+            plog('WARN', e)
             return 0
         except (IndexError, TypeError):
-            plog('ERROR', 'An error occured while negotiating socks5 with Tor')
+            plog('WARN', 'An error occured while negotiating socks5 with Tor (timeout?)')
             return 0
         except:
-            plog('ERROR', 'An unknown SSL error occured')
+            plog('WARN', 'An unknown SSL error occured')
             traceback.print_exc()
             return 0
         
@@ -1227,8 +1229,7 @@
     if do_ssl:
         ssl_nodes = scanner.get_nodes_for_port(443)
         ssl_nodes_n = len(ssl_nodes)
-        # the search for https urls is yet too slow
-        ssl_urls =  ['https://mail.google.com', 'https://addons.mozilla.org', 'https://www.fastmail.fm'] 
+        http_urls = get_urls(wordlist, protocol='https', results_per_type=10, g_results_per_page=20)
         ssl_fail = len(scanner.ssl_fail)
 
         if len(ssl_urls) == 0:
@@ -1287,10 +1288,13 @@
     if not (do_ssl or do_http or do_ssh or do_smtp or do_pop or do_imap or do_dns_basic):
         plog('INFO', 'Done.')
         sys.exit(0)
+        
 
+    # TODO: Do set intersection and reuse nodes for shared tests
+
     # start testing
     while 1:  
-        
+       
         # https test  
         if do_ssl:
             candidates = [x for x in ssl_nodes if ('$' + `x.idhex`) not in scanner.ssl_tested]
@@ -1332,7 +1336,7 @@
                 
             scanner.get_new_circuit()
             ssh_site = random.choice(ssh_urls)
-            scanner.check_ssh(ssh_site)
+            scanner.check_openssh(ssh_site)
  
             ssh_tested_n = len(scanner.ssh_tested)
             if ssh_nodes_n > ssh_tested_n: