[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r18405: {torflow} Woops, was fetching the second non-tor document twice after (torflow/trunk/NetworkScanners)



Author: mikeperry
Date: 2009-02-05 16:35:42 -0500 (Thu, 05 Feb 2009)
New Revision: 18405

Modified:
   torflow/trunk/NetworkScanners/libsoat.py
   torflow/trunk/NetworkScanners/soat.py
Log:

Woops, was fetching the second non-tor document twice after
refactoring. Also add a new storage mechanism for SSL scan.



Modified: torflow/trunk/NetworkScanners/libsoat.py
===================================================================
--- torflow/trunk/NetworkScanners/libsoat.py	2009-02-05 17:52:58 UTC (rev 18404)
+++ torflow/trunk/NetworkScanners/libsoat.py	2009-02-05 21:35:42 UTC (rev 18405)
@@ -84,6 +84,27 @@
     self.cert = cert_file
     self.proto = "ssl"
 
+class SSLDomain:
+  def __init__(self, domain):
+    self.domain = domain
+    # These two could just be sets.Set, but I was kind 
+    # of curious about the logline below.
+    self.cert_map = {}
+    self.ip_map = {}
+
+  def add(self, cert_string, ip):
+    if self.ip_map[ip] != cert_string:
+      plog("NOTICE", self.domain+" is rotating certs for IP "+ip+". Interesting..")
+    self.cert_map[cert_string] = ip
+    self.ip_map[ip] = cert_string
+
+  def matches(self, cert_string):
+    return cert_string in self.cert_map
+
+  def seen_ip(self, ip):
+    return ip in self.ip_map
+
+
 class HttpTestResult(TestResult):
   ''' Represents the result of a http test '''
   def __init__(self, exit_node, website, status, reason=None, 

Modified: torflow/trunk/NetworkScanners/soat.py
===================================================================
--- torflow/trunk/NetworkScanners/soat.py	2009-02-05 17:52:58 UTC (rev 18404)
+++ torflow/trunk/NetworkScanners/soat.py	2009-02-05 21:35:42 UTC (rev 18405)
@@ -402,7 +402,7 @@
       if r.site == address:
         kill_results.append(r)
     for r in kill_results:
-      r.mark_false_positive()
+      r.mark_false_positive(reason)
       self.results.remove(r)
     
   def register_exit_failure(self, address, exit_node):
@@ -447,11 +447,9 @@
     ''' check whether a http connection to a given address is molested '''
     plog('INFO', 'Conducting an http test with destination ' + address)
 
-
     # an address representation acceptable for a filename 
     address_file = self.datahandler.safeFilename(address[7:])
     content_prefix = http_content_dir+address_file
-    failed_prefix = http_failed_dir+address_file
     
     # Keep a copy of the cookie jar before mods for refetch or
     # to restore on errors that cancel a fetch
@@ -469,6 +467,9 @@
         sha1sum.update(buf)
         buf = content_file.read(4096)
       content_file.close()
+      
+      added_cookie_jar = cookielib.MozillaCookieJar()
+      added_cookie_jar.load(content_prefix+'.cookies')
       self.cookie_jar.load(content_prefix+'.cookies')
       content = None 
 
@@ -498,10 +499,10 @@
       
       # Need to do set subtraction and only save new cookies.. 
       # or extract/make_cookies
-      new_cookie_jar = cookielib.MozillaCookieJar()
-      for cookie in new_cookies: new_cookie_jar.set_cookie(cookie)
+      added_cookie_jar = cookielib.MozillaCookieJar()
+      for cookie in new_cookies: added_cookie_jar.set_cookie(cookie)
       try:
-        new_cookie_jar.save(content_prefix+'.cookies')
+        added_cookie_jar.save(content_prefix+'.cookies')
       except:
         traceback.print_exc()
         plog("WARN", "Error saving cookies in "+str(self.cookie_jar)+" to "+content_prefix+".cookies")
@@ -514,7 +515,6 @@
       self.tor_cookie_jar = orig_tor_cookie_jar
       return TEST_INCONCLUSIVE
 
-
     defaultsocket = socket.socket
     socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, tor_host, tor_port)
     socket.socket = socks.socksocket
@@ -584,32 +584,16 @@
 
     sha1sum_new = sha.sha(content_new)
 
-    # compare the new and old content
-    # if they match, means the node has been changing the content
-    if sha1sum.hexdigest() == sha1sum_new.hexdigest():
-      # XXX: Check for existence of this file before overwriting
-      exit_content_file = open(failed_prefix+'.content.'+exit_node[1:], 'w')
-      exit_content_file.write(pcontent)
-      exit_content_file.close()
+    if sha1sum.hexdigest() != sha1sum_new.hexdigest():
+      # if content has changed outside of tor, update the saved file
+      os.rename(content_prefix+'.content', content_prefix+'.content-old')
+      new_content_file = open(content_prefix+'.content', 'w')
+      new_content_file.write(content_new)
+      new_content_file.close()
 
-      result = HttpTestResult(exit_node, address, TEST_FAILURE, 
-                              FAILURE_EXITONLY, sha1sum.hexdigest(), 
-                              psha1sum.hexdigest(), content_prefix+".content",
-                              exit_content_file.name)
-      self.results.append(result)
-      self.datahandler.saveResult(result)
-
-      self.register_exit_failure(address, exit_node)
-      return TEST_FAILURE
-
-    # if content has changed outside of tor, update the saved file
-    os.rename(content_prefix+'.content', content_prefix+'.content-old')
-    new_content_file = open(content_prefix+'.content', 'w')
-    new_content_file.write(content_new)
-    new_content_file.close()
-
     # Need to do set subtraction and only save new cookies.. 
     # or extract/make_cookies
+    self.cookie_jar = orig_cookie_jar
     new_cookie_jar = cookielib.MozillaCookieJar()
     for cookie in new_cookies_new: new_cookie_jar.set_cookie(cookie)
     os.rename(content_prefix+'.cookies', content_prefix+'.cookies-old')
@@ -649,6 +633,24 @@
     content_prefix = http_content_dir+address_file
     failed_prefix = http_failed_dir+address_file
 
+    # compare the new and old content
+    # if they match, means the node has been changing the content
+    if sha1sum.hexdigest() == sha1sum_new.hexdigest():
+      # XXX: Check for existence of this file before overwriting
+      exit_content_file = open(failed_prefix+'.content.'+exit_node[1:], 'w')
+      exit_content_file.write(pcontent)
+      exit_content_file.close()
+
+      result = HttpTestResult(exit_node, address, TEST_FAILURE, 
+                              FAILURE_EXITONLY, sha1sum.hexdigest(), 
+                              psha1sum.hexdigest(), content_prefix+".content",
+                              exit_content_file.name)
+      self.results.append(result)
+      self.datahandler.saveResult(result)
+
+      self.register_exit_failure(address, exit_node)
+      return TEST_FAILURE
+
     # XXX: Check for existence of this file before overwriting
     exit_content_file = open(failed_prefix+'.dyn-content.'+exit_node[1:], 'w')
     exit_content_file.write(pcontent)
@@ -819,9 +821,9 @@
 
     jsdiff = JSDiffer(orig_js)
     jsdiff.prune_differences(new_js)
-    false_positive = not jsdiff.contains_differences(tor_js)
+    has_js_changes = jsdiff.contains_differences(tor_js)
 
-    if false_positive:
+    if not has_js_changes:
       result = JsTestResult(exit_node, address, TEST_SUCCESS)
       self.results.append(result)
       #self.datahandler.saveResult(result)
@@ -851,10 +853,6 @@
   def check_html(self, address):
     plog('INFO', 'Conducting an html test with destination ' + address)
 
-    # Keep a copy of the cookie jar before mods for refetch
-    orig_cookie_jar = cookielib.MozillaCookieJar()
-    for cookie in self.cookie_jar: orig_cookie_jar.set_cookie(cookie)
-
     ret = self.check_http_nodynamic(address)
     
     if type(ret) == int:
@@ -893,13 +891,6 @@
       else: self.successes[address]=1
       return TEST_SUCCESS
 
-    # if content doesnt match, update the direct content and use new cookies
-    # If we have alternate IPs to bind to on this box, use them?
-    # Sometimes pages have the client IP encoded in them..
-    BindingSocket.bind_to = refetch_ip
-    (code_new, new_cookies_new, mime_type_new, new_html) = http_request(address, orig_cookie_jar, self.headers)
-    BindingSocket.bind_to = None
-
     content_new = new_html.decode('ascii', 'ignore')
     if not content_new:
       plog("WARN", "Failed to re-frech "+address+" outside of Tor. Did our network fail?")
@@ -909,6 +900,7 @@
       self.datahandler.saveResult(result)
       return TEST_INCONCLUSIVE
 
+
     new_soup = self._recursive_strain(BeautifulSoup(content_new,
                                      parseOnlyThese=elements))
     # compare the new and old content
@@ -928,23 +920,6 @@
       self.register_exit_failure(address, exit_node)
       return TEST_FAILURE
 
-    # if content has changed outside of tor, update the saved files
-    os.rename(content_prefix+'.content', content_prefix+'.content-old')
-    new_content_file = open(content_prefix+'.content', 'w')
-    new_content_file.write(new_html)
-    new_content_file.close()
-      
-    os.rename(content_prefix+'.cookies', content_prefix+'.cookies-old')
-    # Need to do set subtraction and only save new cookies.. 
-    # or extract/make_cookies
-    new_cookie_jar = cookielib.MozillaCookieJar()
-    for cookie in new_cookies_new: new_cookie_jar.set_cookie(cookie)
-    try:
-      new_cookie_jar.save(content_prefix+'.cookies')
-    except:
-      traceback.print_exc()
-      plog("WARN", "Error saving cookies in "+str(new_cookie_jar)+" to "+content_prefix+".cookies")
-
     # Lets try getting just the tag differences
     # 1. Take difference between old and new tags both ways
     # 2. Make map of tags that change to their attributes
@@ -992,7 +967,7 @@
     exit_content_file.close()
 
     result = HtmlTestResult(exit_node, address, TEST_FAILURE, 
-                            FAILURE_DYNAMICTAGS, new_content_file.name,
+                            FAILURE_DYNAMICTAGS, content_prefix+".content",
                             exit_content_file.name, 
                             content_prefix+'.content-old')
     self.results.append(result)
@@ -1029,7 +1004,8 @@
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     c = SSL.Connection(ctx, s)
     c.set_connect_state()
-     
+  
+    # FIXME: Change this whole test to store pickled SSLDomains
     try:
       c.connect((address, 443)) # XXX: Verify TorDNS here too..
       c.send(crypto.dump_certificate_request(crypto.FILETYPE_PEM,request))
@@ -1083,11 +1059,13 @@
     # if we don't have the original cert yet, get it
     original_cert = 0
     try:
+      # XXX: Use pickle with IP:cert string
       cert_file = open(ssl_certs_dir + address_file + '.pem', 'r')
       cert_string = cert_file.read()
       original_cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_string)
     except IOError:
       plog('INFO', 'Opening a direct ssl connection to ' + address)
+      # XXX: Connect to specific IP used via Non-Tor
       original_cert = self.ssl_request(address)
       if not original_cert:
         plog('WARN', 'Error getting the correct cert for ' + address)