[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] First stab at implementing the web_connectivity test
commit a710467d95627c196f78f553e11055e16d9b9e7f
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date: Mon Feb 1 17:29:08 2016 +0100
First stab at implementing the web_connectivity test
---
ooni/nettests/blocking/web_connectivity.py | 217 +++++++++++++++++++++++++++++
1 file changed, 217 insertions(+)
diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
new file mode 100644
index 0000000..bcc3a28
--- /dev/null
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -0,0 +1,217 @@
+# -*- encoding: utf-8 -*-
+
+import json
+from urlparse import urlparse
+
+from twisted.internet import reactor
+from twisted.internet.protocol import Factory, Protocol
+from twisted.internet.endpoints import TCP4ClientEndpoint
+
+from twisted.internet import defer
+from twisted.python import usage
+
+from ooni.utils import log
+
+from ooni.utils.net import StringProducer, BodyReceiver
+from ooni.templates import httpt, dnst, tcpt
+from ooni.errors import failureToString
+
+class TCPConnectProtocol(Protocol):
+ def connectionMade(self):
+ self.transport.loseConnection()
+
+class TCPConnectFactory(Factory):
+ def buildProtocol(self, addr):
+ return TCPConnectProtocol()
+
+
+class UsageOptions(usage.Options):
+ optParameters = [
+ ['url', 'u', None, 'Specify a single URL to test'],
+ ['dns-discovery', 'd', None, 'Specify the dns discovery test helper'],
+ ['backend', 'b', None, 'The web_consistency backend test helper'],
+ ]
+
+
+class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
+ """
+ Web connectivity
+ """
+ name = "Web connectivity"
+ description = ("Performs a HTTP GET request over Tor and one over the "
+ "local network and compares the two results.")
+ author = "Arturo Filastò"
+ version = "0.1.0"
+
+ usageOptions = UsageOptions
+
+ inputFile = [
+ 'file', 'f', None, 'List of URLS to perform GET requests to'
+ ]
+
+ requiredTestHelpers = {
+ 'backend': 'web_connectivity',
+ 'dns-discovery': 'dns_discovery'
+ }
+ requiresRoot = False
+ requiresTor = False
+
+ # Factor used to determine HTTP blockpage detection
+ factor = 0.8
+
+ def setUp(self):
+ """
+ Check for inputs.
+ """
+ if self.localOptions['url']:
+ self.input = self.localOptions['url']
+ if not self.input:
+ raise Exception("No input specified")
+
+ self.report['client_resolver'] = None
+ self.report['dns_consistency'] = None
+ self.report['body_length_match'] = None
+ self.report['accessible'] = None
+ self.report['blocking'] = None
+
+ self.report['tcp_connect'] = [
+ ]
+
+ self.hostname = urlparse(self.input).netloc
+ if not self.hostname:
+ raise Exception("Invalid input")
+
+ self.control = {
+ 'tcp_connect': {},
+ 'dns_consistency': [],
+ 'http_requests': {
+ 'body_length': None,
+ 'headers': {}
+ }
+ }
+
+ def dns_discovery(self):
+ return self.performALookup(self.localOptions['dns-discovery'])
+
+ def dns_consistency(self):
+ return self.performALookup(self.hostname)
+
+ def tcp_connect(self, socket):
+ ip_address, port = socket.split(":")
+ port = int(port)
+ result = {
+ 'ip': ip_address,
+ 'port': port,
+ 'status': {
+ 'success': None,
+ 'failure': None,
+ 'blocked': None
+ }
+ }
+ point = TCP4ClientEndpoint(reactor, ip_address, port)
+ d = point.connect(TCPConnectFactory())
+ @d.addCallback
+ def cb(p):
+ result['status']['success'] = True
+ result['status']['blocked'] = False
+ self.report['tcp_connect'].append(result)
+
+ @d.addErrback
+ def eb(failure):
+ result['status']['success'] = False
+ self.report['tcp_connect'].append(result)
+ return d
+
+ @defer.inlineCallbacks
+ def control_request(self, sockets):
+ bodyProducer = StringProducer(json.dumps({
+ 'http_request': self.input,
+ 'tcp_connect': sockets
+ }))
+ response = yield self.agent.request("POST",
+ str(self.localOptions['backend']),
+ bodyProducer=bodyProducer)
+ try:
+ content_length = int(response.headers.getRawHeaders('content-length')[0])
+ except Exception:
+ content_length = None
+
+ finished = defer.Deferred()
+ response.deliverBody(BodyReceiver(finished, content_length))
+ body = yield finished
+ self.control = json.loads(body)
+
+ def experiment_http_get_request(self):
+ return self.doRequest(self.input)
+
+ def compare_control_experiment(self, experiment_http_response,
+ experiment_dns_answers):
+
+ blocking = None
+
+ control_body_length = self.control['http_request']['body_length']
+ experiment_body_length = len(experiment_http_response.body)
+
+ if control_body_length == experiment_body_length:
+ rel = float(1)
+ elif control_body_length == 0 or experiment_body_length == 0:
+ rel = float(0)
+ else:
+ rel = float(control_body_length) / float(experiment_body_length)
+
+ if rel > 1:
+ rel = 1/rel
+
+ self.report['body_proportion'] = rel
+ if rel > float(self.factor):
+ self.report['body_length_match'] = True
+ else:
+ blocking = 'http'
+ self.report['body_length_match'] = False
+
+ control_ips = set(self.control['dns']['ips'])
+ experiment_ips = set(experiment_dns_answers)
+
+ if len(control_ips.intersection(experiment_ips)) > 0:
+ self.report['dns_consistency'] = 'consistent'
+ else:
+ if blocking is not None:
+ blocking = 'dns'
+ self.report['dns_consistency'] = 'inconsistent'
+
+ for idx, result in enumerate(self.report['tcp_connect']):
+ socket = "%s:%s" % (result['ip'], result['port'])
+ control_status = self.control['tcp_connect'][socket]
+ log.debug(str(result))
+ if result['status']['success'] == False and \
+ control_status['status'] == True:
+ self.report['tcp_connect'][idx]['status']['blocked'] = True
+ blocking = 'tcp_ip'
+ else:
+ self.report['tcp_connect'][idx]['status']['blocked'] = False
+
+ self.report['blocking'] = blocking
+
+ @defer.inlineCallbacks
+ def test_web_connectivity(self):
+ results = yield defer.DeferredList([
+ self.dns_discovery(),
+ self.dns_consistency()
+ ])
+
+ self.report['client_resolver'] = None
+ if results[0][0] == True:
+ self.report['client_resolver'] = results[0][1][1]
+
+ experiment_dns_answers = results[1][1]
+
+ sockets = map(lambda x: "%s:80" % x, results[1][1])
+
+ dl = [self.control_request(sockets)]
+ for socket in sockets:
+ dl.append(self.tcp_connect(socket))
+ yield defer.DeferredList(dl)
+
+ experiment_http_response = yield self.experiment_http_get_request()
+ self.compare_control_experiment(experiment_http_response,
+ experiment_dns_answers)
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits