[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [torflow/master] Incorporating ssl test into new infrastructure



commit 5ae4e364f809169c19add390e64053bd834e216e
Author: christian <christian@xxxxxxxxx>
Date:   Tue Aug 30 14:03:28 2011 -0400

    Incorporating ssl test into new infrastructure
---
 NetworkScanners/ExitAuthority/soat.py |  142 ++++++++++++++++----------------
 1 files changed, 71 insertions(+), 71 deletions(-)

diff --git a/NetworkScanners/ExitAuthority/soat.py b/NetworkScanners/ExitAuthority/soat.py
index 373dc00..4633dad 100755
--- a/NetworkScanners/ExitAuthority/soat.py
+++ b/NetworkScanners/ExitAuthority/soat.py
@@ -32,7 +32,6 @@ import getopt
 import httplib
 import mimetypes
 import os
-import pickle
 import random
 import re
 import signal
@@ -447,7 +446,7 @@ def ssl_request(address):
   try:
     return _ssl_request(address)
   except (ReadTimeout, socket.timeout), e:
-    plog("INFO", "SSL Request done with timoeut for addrress: "+str(address))
+    plog("INFO", "SSL Request done with timoeut for address: "+str(address))
     return (E_TIMEOUT, None, "Socket timeout")
 
 def _ssl_request(address, method='TLSv1_METHOD'):
@@ -639,7 +638,7 @@ class Test:
       self.url_reserve = {}
       self._pickle_revision = 8
 
-  def _is_useable_url(self, url, valid_schemes=None, filetypes=None):
+  def _is_useable_url(self, url, valid_schemes=None, filetype=None):
     (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url)
     if netloc.rfind(":") != -1:
       # FIXME: %-encoding?
@@ -658,12 +657,10 @@ class Test:
     if url in self.banned_targets:
       plog("DEBUG", "Banned url "+url)
       return False
-    if filetypes: # Must be checked last
-      for filetype in filetypes:
-        if url[-len(filetype):] == filetype:
-          return True
-      plog("DEBUG", "Bad filetype for "+url)
-      return False
+    if filetype:
+      if url[-len(filetype):] != filetype:
+        plog("DEBUG", "Bad filetype for "+url)
+        return False
     return True
 
   def add_target(self, target):
@@ -1010,9 +1007,8 @@ class Test:
     return TEST_INCONCLUSIVE
 
 class BaseHTTPTest(Test):
-  def __init__(self, scan_filetypes=scan_filetypes):
+  def __init__(self):
     # FIXME: Handle http urls w/ non-80 ports..
-    self.scan_filetypes = scan_filetypes
     self.fetch_queue = []
     Test.__init__(self, "HTTP", 80)
     self.save_name = "HTTPTest"
@@ -1172,7 +1168,7 @@ class BaseHTTPTest(Test):
       plog('DEBUG', 'Wrong filetype: ' + loaded_filetype + ' instead of ' + filetype)
       return (address, False, req.code, '')
 
-    self.save_compare_data(address,filetype,req)
+    self.save_compare_data(address,loaded_filetype,req)
 
     # Fetch again with different cookies and see if we get the same content
     # Use a different IP address if possible
@@ -1188,7 +1184,7 @@ class BaseHTTPTest(Test):
       plog("NOTICE", "Non-tor HTTP error "+str(second_req.code)+" fetching content for "+address)
       return (address, False, second_req.code, '')
 
-    if self.compare(address,filetype,second_req) != COMPARE_EQUAL:
+    if self.compare(address,loaded_filetype,second_req) != COMPARE_EQUAL:
       return (address, False, second_req.code, '')
 
     return (address, True, req.code, loaded_filetype)    
@@ -1278,7 +1274,7 @@ class BaseHTTPTest(Test):
       result = HttpTestResult(self.node_map[exit_node[1:]],
                               address, TEST_FAILURE, fail_reason)
       if extra_info:
-        result.extra_info = str(pcontent)
+        result.extra_info = str(preq.content)
 
       return register(result)
 
@@ -1376,9 +1372,8 @@ class BaseHTTPTest(Test):
       working_hash.update(l)
       hashes.append(working_hash.hexdigest())
 
-    f = open(context + '.hashes','w')
-    pickle.dump(hashes,f)
-    f.close()
+    # Save these line-by-line hashes for later use
+    SnakePickler.dump(hashes, context + '.hashes')
 
     # Save the response headers in case we want them for a later test
     headerdiffer = HeaderDiffer(req.headers)
@@ -1400,7 +1395,8 @@ class BaseHTTPTest(Test):
     old_content = f.read()
     f.close()
 
-    old_hashes = pickled_content(context,'.hashes')
+    
+    old_hashes = SnakePickler.load(context + '.hashes')
 
     if len(new_linelist) > len(old_hashes):
       retval = COMPARE_NOEQUAL
@@ -1449,15 +1445,6 @@ class BaseHTTPTest(Test):
       return COMPARE_NOEQUAL
 
 # TODO move these somewhere sensible
-def pickled_content(context,extension):
-  try:
-    f = open(context + extension, 'r')
-    ret = pickle.load(f)
-    f.close()
-  except IOError:
-    ret = False
-  return ret
-
 def mime_to_filetype(mime_type):
   return mimetypes.guess_extension(mime_type)[1:]
 
@@ -1478,8 +1465,8 @@ def is_script_mimetype(mime_type):
   return is_script
 
 class BaseHTMLTest(BaseHTTPTest):
-  def __init__(self, scan_filetypes=scan_filetypes):
-    BaseHTTPTest.__init__(self, scan_filetypes)
+  def __init__(self):
+    BaseHTTPTest.__init__(self)
     self.save_name = "HTMLTest"
     self.proto = "HTML" #CA .. ?
 
@@ -1833,7 +1820,7 @@ class BaseSSLTest(Test):
 
   def run_test(self):
     self.tests_run += 1
-    return self.check_ssl(random.choice(self.targets))
+    return self.check_ssl(random.choice(self.targets)[0])
 
   def get_resolved_ip(self, hostname):
     # XXX: This is some extreme GIL abuse.. It may have race conditions
@@ -2043,11 +2030,12 @@ class FixedTargetHTTPTest(FixedTargetTest, BaseHTTPTest):
     utargets = [t for t in targets if self._is_useable_url(t, ['http'])]
     FixedTargetTest.__init__(self, utargets)
 
-  def get_targets(self):
+  def get_targets(self): 
     ret = []
     for targ in self.fixed_targets:
       addr, succ, code, ftype = self.first_load(targ, False)
-      if succ: ret.append([addr,ftype])
+      if succ: 
+        ret.append([addr,ftype])
     return ret
 
   def add_target(self, target):
@@ -2065,6 +2053,7 @@ class FixedTargetHTMLTest(FixedTargetTest, BaseHTMLTest):
 class FixedTargetSSLTest(FixedTargetTest, BaseSSLTest):
   def __init__(self, targets):
     BaseSSLTest.__init__(self)
+    # We ask for hostnames only, please
     utargets = [t for t in targets if self._is_useable_url(t, [''])]
     FixedTargetTest.__init__(self, utargets)
 
@@ -2080,30 +2069,7 @@ class SearchBasedTest:
   def rewind(self):
     self.wordlist = load_wordlist(self.wordlist_file)
 
-  def add_target(self, target):
-    self.targets.add(target[0],[target[1]])
-    return True
-
-  def get_targets(self):
-    '''
-    construct a list of urls based on the wordlist, filetypes and protocol.
-    '''
-    plog('INFO', 'Searching for relevant sites...')
-
-    urllist = set([])
-    for filetype in self.scan_filetypes:
-      urllist.update(map(lambda x: (x, filetype), self.get_search_urls_for_filetype(filetype)))
-
-    return list(urllist)
-
-  def get_search_urls_for_filetype(self, filetype, number=0):
-    # We don't want to support 'any' any more. We must specify a filetype
-    assert(filetype != 'any')
-    assert(filetype)
-
-    if not number:
-      number = self.results_per_type
-
+  def get_search_urls_for_filetype(self, filetype, number):
     self.url_reserve.setdefault(filetype,[])
 
     type_urls = set(self.url_reserve[filetype][:number])
@@ -2116,7 +2082,7 @@ class SearchBasedTest:
 
       #Try to filter based on filetype/protocol. Unreliable. We will re-filter.
       query = random.choice(self.wordlist)
-      if filetype != 'any':
+      if filetype:
         query += " "+self.search_mode["filetype"]+filetype
       plog("WARN", "RESULTPROTOCOL IS:" + self.result_protocol)
       if self.result_protocol == 'https' and self.search_mode["inurl"]:
@@ -2183,17 +2149,18 @@ class SearchBasedTest:
           prot_list = None
         else:
           prot_list = [self.result_protocol]
-        if filetype == 'any':
-          file_list = None
-        else:
-          file_list = self.scan_filetypes
 
-        if self._is_useable_url(url, prot_list, file_list):
+        if self._is_useable_url(url, prot_list, filetype):
+          try:
+            self.first_load
+          except AttributeError:
+            pass
+          else:
+            url, success, code, cur_filetype = self.first_load(url,filetype)
+            if not success:
+              continue
+
           plog('DEBUG', "Found a useable url: " + url)
-          url, success, code, cur_filetype = self.first_load(url,filetype)
-          if not success:
-            plog('DEBUG',"Url was not useable after all: " + url)
-            continue
           if self.host_only:
             # FIXME: %-encoding, @'s, etc?
             plog("INFO", url)
@@ -2217,9 +2184,10 @@ class SearchBasedTest:
     return type_urls
 
 class SearchBasedHTTPTest(SearchBasedTest, BaseHTTPTest):
-  def __init__(self, wordlist):
+  def __init__(self, wordlist, scan_filetypes=scan_filetypes):
     BaseHTTPTest.__init__(self)
     SearchBasedTest.__init__(self, wordlist)
+    self.scan_filetypes = scan_filetypes
     self.results_per_type = urls_per_filetype
     self.result_protocol = 'http'
 
@@ -2250,6 +2218,22 @@ class SearchBasedHTTPTest(SearchBasedTest, BaseHTTPTest):
         plog("NOTICE", self.proto+" scanner short on "+ftype+" targets. Adding more")
         map(self.add_target, self.get_search_urls_for_filetype(ftype,targets_needed))
 
+  def add_target(self, target):
+    self.targets.add(target[0],[target[1]])
+    return True
+
+  def get_targets(self):
+    '''
+    construct a list of urls based on the wordlist, filetypes and protocol.
+    '''
+    plog('INFO', 'Searching for relevant sites...')
+
+    urllist = set([])
+    for filetype in self.scan_filetypes:
+      urllist.update(map(lambda x: (x, filetype), self.get_search_urls_for_filetype(filetype, self.results_per_type)))
+
+    return list(urllist)
+
 HTTPTest = SearchBasedHTTPTest # For resuming from old HTTPTest.*.test files
 
 class SearchBasedHTMLTest(SearchBasedTest, BaseHTMLTest):
@@ -2257,7 +2241,6 @@ class SearchBasedHTMLTest(SearchBasedTest, BaseHTMLTest):
     BaseHTMLTest.__init__(self)
     SearchBasedTest.__init__(self, wordlist)
     self.result_filetypes = ["any"]
-    self.results_per_type = self.fetch_targets
 
   def depickle_upgrade(self):
     if self._pickle_revision < 7:
@@ -2278,8 +2261,11 @@ class SearchBasedSSLTest(SearchBasedTest, BaseSSLTest):
     SearchBasedTest.__init__(self, wordlist)
     self.host_only = True
     self.result_protocol = 'https'
-    if default_search_mode == yahoo_search_mode:
-      plog('WARN', 'Yahoo search mode is not suitable for SSLTests. Continuing anyway.')
+    try:
+      if default_search_mode == yahoo_search_mode:
+        plog('WARN', 'Yahoo search mode is not suitable for SSLTests. Continuing anyway.')
+    except NameError:
+      pass
     self.search_mode=default_search_mode
 
   def depickle_upgrade(self):
@@ -2289,8 +2275,19 @@ class SearchBasedSSLTest(SearchBasedTest, BaseSSLTest):
       self.search_mode=google_search_mode
     BaseSSLTest.depickle_upgrade(self)
 
+  def get_targets(self):
+    '''
+    construct a list of urls based on the wordlist, filetypes and protocol.
+    '''
+    plog('INFO', 'Searching for relevant sites...')
+
+    urllist = set([])
+    urllist.update(self.get_search_urls_for_filetype(None, num_ssl_hosts))
+
+    return list(urllist)
+
   def rewind(self):
-    self.wordlist = load_wordlist(self.wordlist_file)
+    SearchBasedTest.rewind(self)
     BaseSSLTest.rewind(self)
 
 SSLTest = SearchBasedSSLTest # For resuming from old SSLTest.*.test files
@@ -3207,6 +3204,9 @@ def main(argv):
         common_nodes &= test.nodes
       scanhdlr._sanity_check(map(lambda id: test.node_map[id],
                                              test.nodes))
+    print "COMMON NODES"
+    print common_nodes
+
     if common_nodes is None:
       common_nodes = set([])
 



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits