[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] r18489: {torflow} Upgrade several inconclusive conditions to full failures. Al (torflow/trunk/NetworkScanners)
Author: mikeperry
Date: 2009-02-11 08:48:09 -0500 (Wed, 11 Feb 2009)
New Revision: 18489
Modified:
torflow/trunk/NetworkScanners/libsoat.py
torflow/trunk/NetworkScanners/soat.py
Log:
Upgrade several inconclusive conditions to full failures.
Also improve SSL result reporting.
Modified: torflow/trunk/NetworkScanners/libsoat.py
===================================================================
--- torflow/trunk/NetworkScanners/libsoat.py 2009-02-11 13:41:53 UTC (rev 18488)
+++ torflow/trunk/NetworkScanners/libsoat.py 2009-02-11 13:48:09 UTC (rev 18489)
@@ -12,9 +12,9 @@
import re
import copy
sys.path.append("./libs")
+from OpenSSL import crypto
from BeautifulSoup.BeautifulSoup import Tag, SoupStrainer
-
import sets
from sets import Set
from soat_config import *
@@ -41,7 +41,6 @@
RESULT_CODES=dict([v,k] for k,v in RESULT_STRINGS.iteritems())
# Inconclusive reasons
-INCONCLUSIVE_NOEXITCONTENT = "InconclusiveNoExitContent"
INCONCLUSIVE_NOLOCALCONTENT = "InconclusiveNoLocalContent"
INCONCLUSIVE_DYNAMICSSL = "InconclusiveDynamicSSL"
INCONCLUSIVE_TORBREAKAGE = "InconclusiveTorBreakage"
@@ -54,11 +53,14 @@
FAILURE_DYNAMICCERTS = "FailureDynamicCerts"
FAILURE_COOKIEMISMATCH = "FailureCookieMismatch"
FAILURE_BADHTTPCODE = "FailureBadHTTPCode"
+FAILURE_MISCEXCEPTION = "FailureMiscException"
+FAILURE_NOEXITCONTENT = "FailureNoExitContent"
# False positive reasons
FALSEPOSITIVE_HTTPERRORS = "FalsePositiveHTTPErrors"
FALSEPOSITIVE_DYNAMIC = "FalsePositiveDynamic"
FALSEPOSITIVE_DYNAMIC_TOR = "FalsePositiveDynamicTor"
+FALSEPOSITIVE_DEADSITE = "FalsePositiveDeadSite"
# classes to use with pickle to dump test results into files
@@ -128,6 +130,13 @@
TestResult.mark_false_positive(self, reason)
self.ssl_file=self.move_file(self.ssl_file, ssl_falsepositive_dir)
+ def _dump_cert(self, cert):
+ ret = ""
+ x509 = crypto.load_certificate(crypto.FILETYPE_PEM, cert)
+ ret += "Issuer: "+str(x509.get_issuer())+"\n"
+ ret += "Subject: "+str(x509.get_subject())+"\n"
+ return ret
+
def __str__(self):
ret = TestResult.__str__(self)
ssl_file = open(self.ssl_file, 'r')
@@ -138,14 +147,16 @@
if self.verbose:
for cert in ssl_domain.cert_map.iterkeys():
ret += "\nCert for "+ssl_domain.cert_map[cert]+":\n"
- ret += cert+"\n"
+ ret += cert
+ ret += self._dump_cert(cert)
if self.exit_cert:
# XXX: Kill the first part of this clause after restart:
if 'exit_ip' in self.__dict__ and self.exit_ip:
ret += "\nExit node's cert for "+self.exit_ip+":\n"
else:
ret += "\nExit node's cert:\n"
- ret += self.exit_cert+"\n"
+ ret += self.exit_cert
+ ret += self._dump_cert(self.exit_cert)
return ret
class SSLDomain:
@@ -172,6 +183,8 @@
def seen_ip(self, ip):
return ip in self.ip_map
+ def num_certs(self):
+ return len(self.cert_map)
class HttpTestResult(TestResult):
''' Represents the result of a http test '''
Modified: torflow/trunk/NetworkScanners/soat.py
===================================================================
--- torflow/trunk/NetworkScanners/soat.py 2009-02-11 13:41:53 UTC (rev 18488)
+++ torflow/trunk/NetworkScanners/soat.py 2009-02-11 13:48:09 UTC (rev 18489)
@@ -141,10 +141,15 @@
plog('WARN', 'The http-request address ' + address + ' is malformed')
traceback.print_exc()
return (0, [], "", "")
- except (IndexError, TypeError, socks.Socks5Error), e:
- plog('NOTICE', 'An error occured while negotiating socks5 with Tor: '+str(e))
- traceback.print_exc()
- return (0, [], "", "")
+ except socks.Socks5Error, e:
+ if e.value[0] == 1 or e.value[0] == 6: # Timeout or 'general'
+ plog('NOTICE', 'An error occured while negotiating socks5 with Tor: '+str(e))
+ traceback.print_exc()
+ return (0, [], "", "")
+ else:
+ plog('WARN', 'An unknown SOCKS5 error occured for '+address+": "+str(e))
+ traceback.print_exc()
+ return (666, [], "", str(e))
except KeyboardInterrupt:
raise KeyboardInterrupt
except Exception, e:
@@ -170,6 +175,10 @@
self.results = []
self.dynamic_fails = {}
self.banned_targets = sets.Set([])
+ self.total_nodes = 0
+ self.nodes = []
+ self.node_map = {}
+ self.all_nodes = sets.Set([])
def run_test(self):
raise NotImplemented()
@@ -254,7 +263,7 @@
# TODO: Do something if abundance of succesful tests?
# Problem is this can still trigger for localized content
err_cnt = len(self.exit_fails[address])
- if err_cnt > self.exit_limit_pct*self.total_nodes/100.0:
+ if self.total_nodes and err_cnt > self.exit_limit_pct*self.total_nodes/100.0:
if address not in self.successes: self.successes[address] = 0
plog("NOTICE", "Excessive "+self.proto+" 2-way failure ("+str(err_cnt)+" vs "+str(self.successes[address])+") for "+address+". Removing.")
@@ -416,6 +425,8 @@
tor_cookies = "\n"
plain_cookies = "\n"
# XXX: do we need to sort these? So far we have worse problems..
+ # We probably only want to do this on a per-url basis.. Then
+ # we can do the 3-way compare..
for cookie in self.tor_cookie_jar:
tor_cookies += "\t"+cookie.name+":"+cookie.domain+cookie.path+" discard="+str(cookie.discard)+"\n"
for cookie in self.cookie_jar:
@@ -484,7 +495,7 @@
self.httpcode_fails[address] = sets.Set([exit_node])
err_cnt = len(self.httpcode_fails[address])
- if err_cnt > self.httpcode_limit_pct*self.total_nodes/100.0:
+ if self.total_nodes and err_cnt > self.httpcode_limit_pct*self.total_nodes/100.0:
# Remove all associated data for this url.
# (Note, this also seems to imply we should report BadExit in bulk,
# after we've had a chance for these false positives to be weeded out)
@@ -619,15 +630,15 @@
# if we have no content, we had a connection error
if pcontent == "":
- plog("NOTICE", exit_node+" failed to fetch content for "+address)
- result = HttpTestResult(exit_node, address, TEST_INCONCLUSIVE,
- INCONCLUSIVE_NOEXITCONTENT)
+ plog("ERROR", exit_node+" failed to fetch content for "+address)
+ result = HttpTestResult(exit_node, address, TEST_FAILURE,
+ FAILURE_NOEXITCONTENT)
self.results.append(result)
self.datahandler.saveResult(result)
# Restore cookie jars
self.cookie_jar = orig_cookie_jar
self.tor_cookie_jar = orig_tor_cookie_jar
- return TEST_INCONCLUSIVE
+ return TEST_FAILURE
# compare the content
# if content matches, everything is ok
@@ -1058,37 +1069,40 @@
# open an ssl connection
# FIXME: Hrmmm. handshake considerations
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- c = SSL.Connection(ctx, s)
- c.set_connect_state()
-
try:
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ c = SSL.Connection(ctx, s)
+ c.set_connect_state()
c.connect((address, 443)) # DNS OK.
c.send(crypto.dump_certificate_request(crypto.FILETYPE_PEM,request))
except socket.error, e:
plog('WARN','An error occured while opening an ssl connection to '+address+": "+str(e))
- return 0
- except (IndexError, TypeError, socks.Socks5Error), e:
- plog('WARN', 'An error occured while negotiating socks5 for '+address+': '+str(e))
- return 0
+ return e
+ except socks.Socks5Error, e:
+ if e.value[0] == 1 or e.value[0] == 6: # Timeout or 'general'
+ plog('NOTICE', 'An error occured while negotiating socks5 for '+address+': '+str(e))
+ return -1
+ else:
+ plog('WARN', 'An error occured while negotiating socks5 for '+address+': '+str(e))
+ return e
except KeyboardInterrupt:
raise KeyboardInterrupt
except Exception, e:
plog('WARN', 'An unknown SSL error occured for '+address+': '+str(e))
traceback.print_exc()
- return 0
+ return e
# return the cert
return c.get_peer_certificate()
def get_resolved_ip(self, hostname):
- mappings = self.mt.__control.get_address_mappings("cache")
+ mappings = self.mt.control.get_address_mappings("cache")
ret = None
for m in mappings:
- if m.from_name == hostname:
+ if m.from_addr == hostname:
if ret:
plog("WARN", "Multiple maps for "+hostname)
- ret = m.to_name
+ ret = m.to_addr
return ret
def _update_cert_list(self, ssl_domain, check_ips):
@@ -1097,7 +1111,7 @@
if not ssl_domain.seen_ip(ip):
plog('INFO', 'Ssl connection to new ip '+ip+" for "+ssl_domain.domain)
raw_cert = self.ssl_request(ip)
- if not raw_cert:
+ if not raw_cert or isinstance(raw_cert, Exception):
plog('WARN', 'Error getting the correct cert for '+ssl_domain.domain+":"+ip)
continue
ssl_domain.add_cert(ip,
@@ -1165,6 +1179,16 @@
self.remove_target(address, FALSEPOSITIVE_DYNAMIC)
return TEST_INCONCLUSIVE
+ if not ssl_domain.num_certs():
+ plog("NOTICE", "No non-tor certs available for "+address)
+ result = SSLTestResult("NoExit", address, ssl_file_name,
+ TEST_INCONCLUSIVE,
+ INCONCLUSIVE_NOLOCALCONTENT)
+ self.datahandler.saveResult(result)
+ self.results.append(result)
+ self.remove_target(address, FALSEPOSITIVE_DEADSITE)
+ return TEST_INCONCLUSIVE
+
# get the cert via tor
defaultsocket = socket.socket
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, tor_host, tor_port)
@@ -1178,20 +1202,48 @@
exit_node = self.mt.get_exit_node()
if not exit_node or exit_node == '0':
plog('WARN', 'We had no exit node to test, skipping to the next test.')
- return TEST_FAILURE
+ return TEST_INCONCLUSIVE
- # if we got no cert, there was an ssl error
- if cert == 0:
+ if cert == -1:
+ plog('NOTICE', 'SSL test inconclusive for: '+address)
result = SSLTestResult(exit_node, address, ssl_file_name,
TEST_INCONCLUSIVE,
- INCONCLUSIVE_NOEXITCONTENT)
+ INCONCLUSIVE_TORBREAKAGE)
self.datahandler.saveResult(result)
self.results.append(result)
return TEST_INCONCLUSIVE
- # get an easily comparable representation of the certs
- cert_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
+ # if we got no cert, there was an ssl error
+ if not cert:
+ plog('ERROR', 'SSL failure with empty cert for: '+address+' via '+exit_node)
+ result = SSLTestResult(exit_node, address, ssl_file_name,
+ TEST_FAILURE,
+ FAILURE_NOEXITCONTENT)
+ self.datahandler.saveResult(result)
+ self.results.append(result)
+ return TEST_FAILURE
+ if isinstance(cert, Exception):
+ plog('ERROR', 'SSL failure with exception '+str(cert)+' for: '+address+' via '+exit_node)
+ result = SSLTestResult(exit_node, address, ssl_file_name, TEST_FAILURE,
+ FAILURE_MISCEXCEPTION+str(cert))
+ self.results.append(result)
+ self.datahandler.saveResult(result)
+ self.register_dynamic_failure(address, exit_node)
+ return TEST_FAILURE
+
+ try:
+ # get an easily comparable representation of the certs
+ cert_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
+ except OpenSSL.crypto.Error, e:
+ plog('ERROR', 'SSL failure with exception '+str(e)+' for: '+address+' via '+exit_node)
+ result = SSLTestResult(exit_node, address, ssl_file_name, TEST_FAILURE,
+ FAILURE_MISCEXCEPTION+str(e))
+ self.results.append(result)
+ self.datahandler.saveResult(result)
+ self.register_dynamic_failure(address, exit_node)
+ return TEST_FAILURE
+
# if certs match, everything is ok
if ssl_domain.seen_cert(cert_pem):
result = SSLTestResult(exit_node, address, ssl_file_name, TEST_SUCCESS)
@@ -1892,7 +1944,7 @@
s.connect((control_host, control_port))
c = Connection(s)
c.authenticate()
- self.__control = c
+ self.control = c
except socket.error, e:
plog('ERROR', 'Couldn\'t connect to the control port')
plog('ERROR', e)
@@ -1958,7 +2010,7 @@
'''
# get the structure
- routers = self.__control.read_routers(self.__control.get_network_status())
+ routers = self.control.read_routers(self.control.get_network_status())
bad_exits = Set([])
specific_bad_exits = [None]*len(ports_to_check)
for i in range(len(ports_to_check)):
@@ -2195,13 +2247,12 @@
plog("NOTICE", "Scanning only "+scan_exit)
mt.set_new_exit(scan_exit)
mt.get_new_circuit()
+
+ while 1:
+ for test in tests.values():
+ result = test.run_test()
+ plog("INFO", test.proto+" test via "+scan_exit+" has result "+str(result))
- for test in tests.values():
- result = test.run_test()
- plog("INFO", test.proto+" test via "+scan_exit+" has result "+str(result))
- plog('INFO', 'Done.')
- sys.exit(0)
-
# start testing
while 1:
avail_tests = tests.values()