[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Add support for ignoring redirects to IPs in private IP space
commit 4f0c2af37093070e4cec0593a0f57568a4db231c
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date: Wed Nov 23 16:08:27 2016 +0000
Add support for ignoring redirects to IPs in private IP space
---
ooni/common/ip_utils.py | 29 +++++++++++++++++++++++++++++
ooni/common/txextra.py | 18 +++++++++++++++++-
ooni/nettests/blocking/web_connectivity.py | 30 +++++++++---------------------
ooni/templates/httpt.py | 26 +++++++++++++++++++++-----
4 files changed, 76 insertions(+), 27 deletions(-)
diff --git a/ooni/common/ip_utils.py b/ooni/common/ip_utils.py
new file mode 100644
index 0000000..8745b7a
--- /dev/null
+++ b/ooni/common/ip_utils.py
@@ -0,0 +1,29 @@
+from ipaddr import IPv4Address, IPv6Address
+from ipaddr import AddressValueError
+
+
+def is_public_ipv4_address(address):
+ return not is_private_ipv4_address(address)
+
+
+def is_private_ipv4_address(address):
+ try:
+ ip_address = IPv4Address(address)
+ return any(
+ [ip_address.is_private, ip_address.is_loopback]
+ )
+ except AddressValueError:
+ return False
+
+def is_private_address(address):
+ try:
+ ip_address = IPv4Address(address)
+ except AddressValueError:
+ try:
+ ip_address = IPv6Address(address)
+ except AddressValueError:
+ return False
+
+ return any(
+ [ip_address.is_private, ip_address.is_loopback]
+ )
diff --git a/ooni/common/txextra.py b/ooni/common/txextra.py
index 7a84592..39b8996 100644
--- a/ooni/common/txextra.py
+++ b/ooni/common/txextra.py
@@ -19,6 +19,8 @@ from twisted.internet.defer import Deferred, fail, maybeDeferred, failure
from twisted.python import log
+from .ip_utils import is_private_address
+
class TrueHeaders(Headers):
def __init__(self, rawHeaders=None):
self._rawHeaders = dict()
@@ -168,6 +170,10 @@ class FixedRedirectAgent(BrowserLikeRedirectAgent):
This is a redirect agent with this patch manually applied:
https://twistedmatrix.com/trac/ticket/8265
"""
+ def __init__(self, agent, redirectLimit=20, ignorePrivateRedirects=False):
+ self.ignorePrivateRedirects = ignorePrivateRedirects
+ BrowserLikeRedirectAgent.__init__(self, agent, redirectLimit)
+
def _handleRedirect(self, response, method, uri, headers, redirectCount):
"""
Handle a redirect response, checking the number of redirects already
@@ -191,12 +197,22 @@ class FixedRedirectAgent(BrowserLikeRedirectAgent):
response.request.absoluteURI,
locationHeaders[0]
)
+ uri = client.URI.fromBytes(location)
+ if self.ignorePrivateRedirects and (is_private_address(uri.host) or
+ uri.host == "localhost"):
+ return response
+
deferred = self._agent.request(method, location, headers)
def _chainResponse(newResponse):
+ if isinstance(newResponse, Failure):
+ # This is needed to write the response even in case of failure
+ newResponse.previousResponse = response
+ newResponse.requestLocation = location
+ return newResponse
newResponse.setPreviousResponse(response)
return newResponse
- deferred.addCallback(_chainResponse)
+ deferred.addBoth(_chainResponse)
return deferred.addCallback(
self._handleResponse, method, uri, headers, redirectCount + 1)
diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index 1d1b742..629ef1c 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -3,28 +3,24 @@
import csv
from urlparse import urlparse
-from ipaddr import IPv4Address, AddressValueError
-
-from twisted.web.client import GzipDecoder
+from twisted.internet import defer
from twisted.internet import reactor
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.names import client
-
-from twisted.internet import defer
from twisted.python import usage
+from twisted.web.client import GzipDecoder
from ooni import geoip
-from ooni.utils import log
-
from ooni.backend_client import WebConnectivityClient
-
+from ooni.common.http_utils import REQUEST_HEADERS
from ooni.common.http_utils import extractTitle
-from ooni.utils.net import COMMON_SERVER_HEADERS
-from ooni.templates import httpt, dnst
+from ooni.common.ip_utils import is_public_ipv4_address
+from ooni.common.tcp_utils import TCPConnectFactory
from ooni.errors import failureToString
+from ooni.templates import httpt, dnst
+from ooni.utils import log
+from ooni.utils.net import COMMON_SERVER_HEADERS
-from ooni.common.tcp_utils import TCPConnectFactory
-from ooni.common.http_utils import REQUEST_HEADERS
class InvalidControlResponse(Exception):
pass
@@ -42,15 +38,6 @@ class UsageOptions(usage.Options):
]
-def is_public_ipv4_address(address):
- try:
- ip_address = IPv4Address(address)
- return not any(
- [ip_address.is_private, ip_address.is_loopback]
- )
- except AddressValueError:
- return None
-
class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
"""
Web connectivity
@@ -79,6 +66,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
requiresRoot = False
requiresTor = False
followRedirects = True
+ ignorePrivateRedirects = True
# These are the options to be shown on the GUI
simpleOptions = [
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index d1aa1dc..e57463b 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -66,6 +66,10 @@ class HTTPTest(NetTestCase):
randomizeUA = False
followRedirects = False
+ # When this is set to False we will follow redirects pointing to IPs in
+ # rfc1918
+ ignorePrivateRedirects = False
+
# You can specify a list of tuples in the format of (CONTENT_TYPE,
# DECODER)
# For example to support Gzip decoding you should specify
@@ -107,7 +111,10 @@ class HTTPTest(NetTestCase):
if self.followRedirects:
try:
self.control_agent = FixedRedirectAgent(self.control_agent)
- self.agent = FixedRedirectAgent(self.agent)
+ self.agent = FixedRedirectAgent(
+ self.agent,
+ ignorePrivateRedirects=self.ignorePrivateRedirects
+ )
self.report['agent'] = 'redirect'
except:
log.err("Warning! You are running an old version of twisted "
@@ -129,7 +136,8 @@ class HTTPTest(NetTestCase):
def processInputs(self):
pass
- def addToReport(self, request, response=None, response_body=None, failure_string=None):
+ def addToReport(self, request, response=None, response_body=None,
+ failure_string=None, previous_response=None):
"""
Adds to the report the specified request and response.
@@ -176,9 +184,10 @@ class HTTPTest(NetTestCase):
session['failure'] = failure_string
self.report['requests'].append(session)
-
if response and response.previousResponse:
- self.addToReport(request, response.previousResponse,
+ previous_response = response.previousResponse
+ if previous_response:
+ self.addToReport(request, previous_response,
response_body=None,
failure_string=None)
@@ -367,7 +376,14 @@ class HTTPTest(NetTestCase):
else:
log.err("Error performing HTTP request: %s" % request['url'])
failure_string = handleAllFailures(failure)
- self.addToReport(request, failure_string=failure_string)
+ previous_response = None
+ if getattr(failure, "previousResponse", None):
+ previous_response = failure.previousResponse
+ if getattr(failure, "requestLocation", None):
+ request['url'] = failure.requestLocation
+
+ self.addToReport(request, failure_string=failure_string,
+ previous_response=previous_response)
return failure
if use_tor:
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits