[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Add support for computing header difference and follow redirects
commit 8c5fdd48ebb760b72b1b02a83fc96a19f92293c7
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date: Sat Apr 16 14:30:29 2016 +0200
Add support for computing header difference and follow redirects
---
ooni/nettests/blocking/web_connectivity.py | 54 +++++++++++++++++++++---------
1 file changed, 39 insertions(+), 15 deletions(-)
diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index 175c6d3..4b9b6c2 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -81,6 +81,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
}
requiresRoot = False
requiresTor = False
+ followRedirects = True
# Factor used to determine HTTP blockpage detection
factor = 0.8
@@ -112,6 +113,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
self.report['client_resolver'] = self.resolverIp
self.report['dns_consistency'] = None
self.report['body_length_match'] = None
+ self.report['headers_match'] = None
+
self.report['accessible'] = None
self.report['blocking'] = None
@@ -139,6 +142,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
}
def experiment_dns_query(self):
+ log.msg("Doing DNS query for {}".format(self.hostname))
return self.performALookup(self.hostname)
def tcp_connect(self, socket):
@@ -197,6 +201,22 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
def experiment_http_get_request(self):
return self.doRequest(self.input, headers=REQUEST_HEADERS)
+ def compare_headers(self, experiment_http_response):
+ count = 0
+ control_headers_lower = {k.lower(): v for k, v in
+ self.report['control']['http_request']['headers'].items()}
+
+ for header_name, header_value in \
+ experiment_http_response.headers.getAllRawHeaders():
+ try:
+ control_headers_lower[header_name.lower()]
+ except KeyError:
+ log.msg("Did not find the key {}".format(header_name))
+ return False
+ count += 1
+
+ return count == len(self.report['control']['http_request']['headers'])
+
def compare_body_lengths(self, experiment_http_response):
control_body_length = self.control['http_request']['body_length']
experiment_body_length = len(experiment_http_response.body)
@@ -213,10 +233,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
self.report['body_proportion'] = rel
if rel > float(self.factor):
- self.report['body_length_match'] = True
return True
else:
- self.report['body_length_match'] = False
return False
def compare_dns_experiments(self, experiment_dns_answers):
@@ -229,16 +247,13 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
experiment_ips = set(experiment_dns_answers)
if control_ips == experiment_ips:
- self.report['dns_consistency'] = 'consistent'
return True
for experiment_ip in experiment_ips:
if is_public_ipv4_address(experiment_ip) is False:
- self.report['dns_consistency'] = 'inconsistent'
return False
if len(control_ips.intersection(experiment_ips)) > 0:
- self.report['dns_consistency'] = 'consistent'
return True
experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'],
@@ -247,10 +262,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
control_ips))
if len(control_asns.intersection(experiment_asns)) > 0:
- self.report['dns_consistency'] = 'consistent'
return True
- self.report['dns_consistency'] = 'inconsistent'
return False
def compare_tcp_experiments(self):
@@ -268,22 +281,26 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
def determine_blocking(self, experiment_http_response, experiment_dns_answers):
blocking = False
- body_length_match = None
- dns_consistent = None
- tcp_connect = None
if (self.report['http_experiment_failure'] is None and
self.report['control']['http_request']['failure'] is None):
- body_length_match = self.compare_body_lengths(experiment_http_response)
+ self.report['body_length_match'] = self.compare_body_lengths(
+ experiment_http_response)
+ self.report['headers_match'] = self.compare_headers(
+ experiment_http_response)
dns_consistent = self.compare_dns_experiments(experiment_dns_answers)
+ if dns_consistent is True:
+ self.report['dns_consistency'] = 'consistent'
+ else:
+ self.report['dns_consistency'] = 'inconsistent'
tcp_connect = self.compare_tcp_experiments()
if dns_consistent == True and tcp_connect == False:
blocking = 'tcp_ip'
elif (dns_consistent == True and tcp_connect == True and
- body_length_match == False):
+ self.report['body_length_match'] == False):
blocking = 'http'
elif dns_consistent == False:
@@ -302,10 +319,17 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
return []
experiment_dns_answers = yield experiment_dns
+ port = 80
+ parsed_url = urlparse(self.input)
+ if parsed_url.port:
+ port = parsed_url.port
+ elif parsed_url.scheme == 'https':
+ port = 443
+
sockets = []
- for answer in experiment_dns_answers:
- if is_public_ipv4_address(answer) is True:
- sockets.append("%s:80" % answer)
+ for ip_address in experiment_dns_answers:
+ if is_public_ipv4_address(ip_address) is True:
+ sockets.append("{}:{}".format(ip_address, port))
# STEALTH in here we should make changes to make the test more stealth
dl = []
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits