[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] r18405: {torflow} Woops, was fetching the second non-tor document twice after (torflow/trunk/NetworkScanners)
Author: mikeperry
Date: 2009-02-05 16:35:42 -0500 (Thu, 05 Feb 2009)
New Revision: 18405
Modified:
torflow/trunk/NetworkScanners/libsoat.py
torflow/trunk/NetworkScanners/soat.py
Log:
Woops, was fetching the second non-tor document twice after
refactoring. Also add a new storage mechanism for SSL scan.
Modified: torflow/trunk/NetworkScanners/libsoat.py
===================================================================
--- torflow/trunk/NetworkScanners/libsoat.py 2009-02-05 17:52:58 UTC (rev 18404)
+++ torflow/trunk/NetworkScanners/libsoat.py 2009-02-05 21:35:42 UTC (rev 18405)
@@ -84,6 +84,27 @@
self.cert = cert_file
self.proto = "ssl"
+class SSLDomain:
+ def __init__(self, domain):
+ self.domain = domain
+ # These two could just be sets.Set, but I was kind
+ # of curious about the logline below.
+ self.cert_map = {}
+ self.ip_map = {}
+
+ def add(self, cert_string, ip):
+ if self.ip_map[ip] != cert_string:
+ plog("NOTICE", self.domain+" is rotating certs for IP "+ip+". Interesting..")
+ self.cert_map[cert_string] = ip
+ self.ip_map[ip] = cert_string
+
+ def matches(self, cert_string):
+ return cert_string in self.cert_map
+
+ def seen_ip(self, ip):
+ return ip in self.ip_map
+
+
class HttpTestResult(TestResult):
''' Represents the result of a http test '''
def __init__(self, exit_node, website, status, reason=None,
Modified: torflow/trunk/NetworkScanners/soat.py
===================================================================
--- torflow/trunk/NetworkScanners/soat.py 2009-02-05 17:52:58 UTC (rev 18404)
+++ torflow/trunk/NetworkScanners/soat.py 2009-02-05 21:35:42 UTC (rev 18405)
@@ -402,7 +402,7 @@
if r.site == address:
kill_results.append(r)
for r in kill_results:
- r.mark_false_positive()
+ r.mark_false_positive(reason)
self.results.remove(r)
def register_exit_failure(self, address, exit_node):
@@ -447,11 +447,9 @@
''' check whether a http connection to a given address is molested '''
plog('INFO', 'Conducting an http test with destination ' + address)
-
# an address representation acceptable for a filename
address_file = self.datahandler.safeFilename(address[7:])
content_prefix = http_content_dir+address_file
- failed_prefix = http_failed_dir+address_file
# Keep a copy of the cookie jar before mods for refetch or
# to restore on errors that cancel a fetch
@@ -469,6 +467,9 @@
sha1sum.update(buf)
buf = content_file.read(4096)
content_file.close()
+
+ added_cookie_jar = cookielib.MozillaCookieJar()
+ added_cookie_jar.load(content_prefix+'.cookies')
self.cookie_jar.load(content_prefix+'.cookies')
content = None
@@ -498,10 +499,10 @@
# Need to do set subtraction and only save new cookies..
# or extract/make_cookies
- new_cookie_jar = cookielib.MozillaCookieJar()
- for cookie in new_cookies: new_cookie_jar.set_cookie(cookie)
+ added_cookie_jar = cookielib.MozillaCookieJar()
+ for cookie in new_cookies: added_cookie_jar.set_cookie(cookie)
try:
- new_cookie_jar.save(content_prefix+'.cookies')
+ added_cookie_jar.save(content_prefix+'.cookies')
except:
traceback.print_exc()
plog("WARN", "Error saving cookies in "+str(self.cookie_jar)+" to "+content_prefix+".cookies")
@@ -514,7 +515,6 @@
self.tor_cookie_jar = orig_tor_cookie_jar
return TEST_INCONCLUSIVE
-
defaultsocket = socket.socket
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, tor_host, tor_port)
socket.socket = socks.socksocket
@@ -584,32 +584,16 @@
sha1sum_new = sha.sha(content_new)
- # compare the new and old content
- # if they match, means the node has been changing the content
- if sha1sum.hexdigest() == sha1sum_new.hexdigest():
- # XXX: Check for existence of this file before overwriting
- exit_content_file = open(failed_prefix+'.content.'+exit_node[1:], 'w')
- exit_content_file.write(pcontent)
- exit_content_file.close()
+ if sha1sum.hexdigest() != sha1sum_new.hexdigest():
+ # if content has changed outside of tor, update the saved file
+ os.rename(content_prefix+'.content', content_prefix+'.content-old')
+ new_content_file = open(content_prefix+'.content', 'w')
+ new_content_file.write(content_new)
+ new_content_file.close()
- result = HttpTestResult(exit_node, address, TEST_FAILURE,
- FAILURE_EXITONLY, sha1sum.hexdigest(),
- psha1sum.hexdigest(), content_prefix+".content",
- exit_content_file.name)
- self.results.append(result)
- self.datahandler.saveResult(result)
-
- self.register_exit_failure(address, exit_node)
- return TEST_FAILURE
-
- # if content has changed outside of tor, update the saved file
- os.rename(content_prefix+'.content', content_prefix+'.content-old')
- new_content_file = open(content_prefix+'.content', 'w')
- new_content_file.write(content_new)
- new_content_file.close()
-
# Need to do set subtraction and only save new cookies..
# or extract/make_cookies
+ self.cookie_jar = orig_cookie_jar
new_cookie_jar = cookielib.MozillaCookieJar()
for cookie in new_cookies_new: new_cookie_jar.set_cookie(cookie)
os.rename(content_prefix+'.cookies', content_prefix+'.cookies-old')
@@ -649,6 +633,24 @@
content_prefix = http_content_dir+address_file
failed_prefix = http_failed_dir+address_file
+ # compare the new and old content
+ # if they match, means the node has been changing the content
+ if sha1sum.hexdigest() == sha1sum_new.hexdigest():
+ # XXX: Check for existence of this file before overwriting
+ exit_content_file = open(failed_prefix+'.content.'+exit_node[1:], 'w')
+ exit_content_file.write(pcontent)
+ exit_content_file.close()
+
+ result = HttpTestResult(exit_node, address, TEST_FAILURE,
+ FAILURE_EXITONLY, sha1sum.hexdigest(),
+ psha1sum.hexdigest(), content_prefix+".content",
+ exit_content_file.name)
+ self.results.append(result)
+ self.datahandler.saveResult(result)
+
+ self.register_exit_failure(address, exit_node)
+ return TEST_FAILURE
+
# XXX: Check for existence of this file before overwriting
exit_content_file = open(failed_prefix+'.dyn-content.'+exit_node[1:], 'w')
exit_content_file.write(pcontent)
@@ -819,9 +821,9 @@
jsdiff = JSDiffer(orig_js)
jsdiff.prune_differences(new_js)
- false_positive = not jsdiff.contains_differences(tor_js)
+ has_js_changes = jsdiff.contains_differences(tor_js)
- if false_positive:
+ if not has_js_changes:
result = JsTestResult(exit_node, address, TEST_SUCCESS)
self.results.append(result)
#self.datahandler.saveResult(result)
@@ -851,10 +853,6 @@
def check_html(self, address):
plog('INFO', 'Conducting an html test with destination ' + address)
- # Keep a copy of the cookie jar before mods for refetch
- orig_cookie_jar = cookielib.MozillaCookieJar()
- for cookie in self.cookie_jar: orig_cookie_jar.set_cookie(cookie)
-
ret = self.check_http_nodynamic(address)
if type(ret) == int:
@@ -893,13 +891,6 @@
else: self.successes[address]=1
return TEST_SUCCESS
- # if content doesnt match, update the direct content and use new cookies
- # If we have alternate IPs to bind to on this box, use them?
- # Sometimes pages have the client IP encoded in them..
- BindingSocket.bind_to = refetch_ip
- (code_new, new_cookies_new, mime_type_new, new_html) = http_request(address, orig_cookie_jar, self.headers)
- BindingSocket.bind_to = None
-
content_new = new_html.decode('ascii', 'ignore')
if not content_new:
plog("WARN", "Failed to re-frech "+address+" outside of Tor. Did our network fail?")
@@ -909,6 +900,7 @@
self.datahandler.saveResult(result)
return TEST_INCONCLUSIVE
+
new_soup = self._recursive_strain(BeautifulSoup(content_new,
parseOnlyThese=elements))
# compare the new and old content
@@ -928,23 +920,6 @@
self.register_exit_failure(address, exit_node)
return TEST_FAILURE
- # if content has changed outside of tor, update the saved files
- os.rename(content_prefix+'.content', content_prefix+'.content-old')
- new_content_file = open(content_prefix+'.content', 'w')
- new_content_file.write(new_html)
- new_content_file.close()
-
- os.rename(content_prefix+'.cookies', content_prefix+'.cookies-old')
- # Need to do set subtraction and only save new cookies..
- # or extract/make_cookies
- new_cookie_jar = cookielib.MozillaCookieJar()
- for cookie in new_cookies_new: new_cookie_jar.set_cookie(cookie)
- try:
- new_cookie_jar.save(content_prefix+'.cookies')
- except:
- traceback.print_exc()
- plog("WARN", "Error saving cookies in "+str(new_cookie_jar)+" to "+content_prefix+".cookies")
-
# Lets try getting just the tag differences
# 1. Take difference between old and new tags both ways
# 2. Make map of tags that change to their attributes
@@ -992,7 +967,7 @@
exit_content_file.close()
result = HtmlTestResult(exit_node, address, TEST_FAILURE,
- FAILURE_DYNAMICTAGS, new_content_file.name,
+ FAILURE_DYNAMICTAGS, content_prefix+".content",
exit_content_file.name,
content_prefix+'.content-old')
self.results.append(result)
@@ -1029,7 +1004,8 @@
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
c = SSL.Connection(ctx, s)
c.set_connect_state()
-
+
+ # FIXME: Change this whole test to store pickled SSLDomains
try:
c.connect((address, 443)) # XXX: Verify TorDNS here too..
c.send(crypto.dump_certificate_request(crypto.FILETYPE_PEM,request))
@@ -1083,11 +1059,13 @@
# if we don't have the original cert yet, get it
original_cert = 0
try:
+ # XXX: Use pickle with IP:cert string
cert_file = open(ssl_certs_dir + address_file + '.pem', 'r')
cert_string = cert_file.read()
original_cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_string)
except IOError:
plog('INFO', 'Opening a direct ssl connection to ' + address)
+ # XXX: Connect to specific IP used via Non-Tor
original_cert = self.ssl_request(address)
if not original_cert:
plog('WARN', 'Error getting the correct cert for ' + address)