[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Merge branch 'master' of git-rw.torproject.org:ooni-probe
commit 1008bd5c04e9a9fa1f1726fa5474725d2714f37c
Merge: 870d882 2bf3da5
Author: Isis Lovecruft <isis@xxxxxxxxxxxxxx>
Date: Thu Nov 15 00:26:43 2012 +0000
Merge branch 'master' of git-rw.torproject.org:ooni-probe
Conflicts:
ooni/utils/net.py
AUTHORS | 2 +-
CHANGES.yaml | 5 -
HACKING | 141 ++++++------
INSTALL | 1 -
README.md | 123 +++++------
TODO | 105 +--------
before_i_commit.sh | 2 +-
bin/oonib | 41 +++-
bin/ooniprobe | 5 +-
docs/source/index.rst | 51 +----
docs/source/install.rst | 119 ----------
nettests/bridge_reachability/echo.py | 2 +-
nettests/core/captiveportal.py | 23 +-
nettests/core/chinatrigger.py | 9 +-
nettests/core/daphn3.py | 117 +++++++++
nettests/core/http_host.py | 74 +++++-
nettests/core/http_requests.py | 118 +++++++++
nettests/core/http_uk_mobile_networks.py | 85 +++++++
nettests/core/keyword_filtering.py | 52 ++++
nettests/core/squid.py | 2 +-
nettests/core/tcpconnect.py | 46 ++++
nettests/core/traceroute.py | 113 +++++++++
nettests/core/url_list.py | 59 ++++-
nettests/examples/example_scapyt.py | 22 ++-
nettests/simpletest.py | 27 --
ooni/__init__.py | 4 +
ooni/config.py | 20 ++-
ooni/inputunit.py | 19 +-
ooni/kit/daphn3.py | 208 ++++++++++++++++
ooni/lib/rfc3339.py | 283 ----------------------
ooni/lib/txscapy.py | 381 ------------------------------
ooni/nettest.py | 38 ++--
ooni/nodes.py | 24 +-
ooni/ooni-probe.conf | 100 --------
ooni/oonicli.py | 67 ++++--
ooni/protocols/daphn3.py | 311 ------------------------
ooni/reporter.py | 340 +++++++++++++++------------
ooni/runner.py | 117 ++++++----
ooni/templates/httpt.py | 120 ++++------
ooni/templates/scapyt.py | 124 +++--------
ooni/utils/__init__.py | 102 +--------
ooni/utils/date.py | 30 ---
ooni/utils/geodata.py | 31 ++-
ooni/utils/hacks.py | 75 ++++---
ooni/utils/log.py | 26 ++-
ooni/utils/net.py | 146 ++++++++----
ooni/utils/otime.py | 8 +
ooni/utils/txscapy.py | 140 +++++++++++
oonib/__init__.py | 22 ++
oonib/config.py | 43 ++++
oonib/db/__init__.py | 30 +++
oonib/db/tables.py | 123 ++++++++++
oonib/lib/__init__.py | 8 -
oonib/lib/ssl.py | 8 -
oonib/models.py | 122 ++++++++++
oonib/oonibackend.conf.sample | 12 -
oonib/oonibackend.py | 117 +++++-----
oonib/report/api.py | 164 ++++++++++++-
oonib/report/db/__init__.py | 33 ---
oonib/requirements.txt | 24 ++
oonib/runner.py | 25 ++
oonib/testhelpers/__init__.py | 5 +
oonib/testhelpers/daphn3.py | 47 ----
oonib/testhelpers/dns.py | 16 --
oonib/testhelpers/dns_helpers.py | 16 ++
oonib/testhelpers/http_helpers.py | 80 +++++++
oonib/testhelpers/httph.py | 92 -------
oonib/testhelpers/ssl_helpers.py | 9 +
oonib/testhelpers/tcp_helpers.py | 72 ++++++
ooniprobe.conf | 6 +-
requirements.txt | 45 ++--
tests/assets/ipports.txt | 2 -
tests/assets/urllist.txt | 3 -
tests/legacy/test_plugins.py | 40 ---
tests/legacy/test_tests.py | 45 ----
tests/test_mutate.py | 15 ++
tests/test_worker.py | 29 ---
to-be-ported/old-api/daphn3.py | 152 ------------
to-be-ported/old-api/httpt.py | 94 --------
to-be-ported/old-api/tcpconnect.py | 65 -----
80 files changed, 2696 insertions(+), 2926 deletions(-)
diff --cc ooni/utils/net.py
index 4d8907d,3ddba61..2583c74
--- a/ooni/utils/net.py
+++ b/ooni/utils/net.py
@@@ -2,169 -2,91 +2,221 @@@
#
# net.py
# --------
- # OONI utilities for network infrastructure and hardware.
- #
- # :authors: Isis Lovecruft, Arturo Filasto
- # :version: 0.0.1-pre-alpha
- # :license: (c) 2012 Isis Lovecruft, Arturo Filasto
- # see attached LICENCE file
+ # OONI utilities for networking related operations
- from pprint import pprint
- from sys import platform
+ import sys
-from zope.interface import implements
- #if platformm.system() == 'Windows':
- # import _winreg as winreg
++from zope.interface import implements
+ from twisted.internet import protocol, defer
+ from twisted.internet import threads, reactor
+ from twisted.web.iweb import IBodyProducer
- from ooni.utils import log
+ from scapy.all import utils
- PLATFORMS = {'LINUX': platform.startswith("linux"),
- 'OPENBSD': platform.startswith("openbsd"),
- 'FREEBSD': platform.startswith("freebsd"),
- 'NETBSD': platform.startswith("netbsd"),
- 'DARWIN': platform.startswith("darwin"),
- 'SOLARIS': platform.startswith("sunos"),
- 'WINDOWS': platform.startswith("win32")}
+ from ooni.utils import log, txscapy
++#if sys.platformm.system() == 'Windows':
++# import _winreg as winreg
+
- class PlatformNameException(Exception):
- """Specified platform does not match client platform."""
+ userAgents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"),
+ ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"),
+ ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"),
+ ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"),
+ ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"),
+ ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"),
+ ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"),
+ ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"),
+ ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"),
+ ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"),
+ ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")]
+
++PLATFORMS = {'LINUX': sys.platform.startswith("linux"),
++ 'OPENBSD': sys.platform.startswith("openbsd"),
++ 'FREEBSD': sys.platform.startswith("freebsd"),
++ 'NETBSD': sys.platform.startswith("netbsd"),
++ 'DARWIN': sys.platform.startswith("darwin"),
++ 'SOLARIS': sys.platform.startswith("sunos"),
++ 'WINDOWS': sys.platform.startswith("win32")}
+
+class UnsupportedPlatform(Exception):
+ """Support for this platform is not currently available."""
+
+class IfaceError(Exception):
+ """Could not find default network interface."""
+
+class PermissionsError(SystemExit):
+ """This test requires admin or root privileges to run. Exiting..."""
+
- def getClientAddress():
- address = {'asn': 'REPLACE_ME',
- 'ip': 'REPLACE_ME'}
- return address
+ class StringProducer(object):
+ implements(IBodyProducer)
+
+ def __init__(self, body):
+ self.body = body
+ self.length = len(body)
+
+ def startProducing(self, consumer):
+ consumer.write(self.body)
+ return defer.succeed(None)
+
+ def pauseProducing(self):
+ pass
+
+ def stopProducing(self):
+ pass
+
+ class BodyReceiver(protocol.Protocol):
+ def __init__(self, finished):
+ self.finished = finished
+ self.data = ""
+
+ def dataReceived(self, bytes):
+ self.data += bytes
+
+ def connectionLost(self, reason):
+ self.finished.callback(self.data)
+
+ def capturePackets(pcap_filename):
+ from scapy.all import sniff
+ global stop_packet_capture
+ stop_packet_capture = False
+
+ def stopCapture():
+ # XXX this is a bit of a hack to stop capturing packets when we close
+ # the reactor. Ideally we would want to be able to do this
+ # programmatically, but this requires some work on implementing
+ # properly the sniff function with deferreds.
+ global stop_packet_capture
+ stop_packet_capture = True
+
+ def writePacketToPcap(pkt):
+ from scapy.all import utils
+ pcapwriter = txscapy.TXPcapWriter(pcap_filename, append=True)
+ pcapwriter.write(pkt)
+ if stop_packet_capture:
+ sys.exit(1)
-
+ d = threads.deferToThread(sniff, lfilter=writePacketToPcap)
+ reactor.addSystemEventTrigger('before', 'shutdown', stopCapture)
+ return d
+
-class PermissionsError(SystemExit):
- def __init__(self, *args, **kwargs):
- if not args and not kwargs:
- pe = "This test requires admin or root privileges to run. Exiting..."
- super(PermissionsError, self).__init__(pe, *args, **kwargs)
- else:
- super(PermissionsError, self).__init__(*args, **kwargs)
-class IfaceError(SystemExit):
- def __init__(self, *args, **kwargs):
- super(IfaceError, self).__init__(*args, **kwargs)
+def getClientPlatform(platform_name=None):
+ for name, test in PLATFORMS.items():
+ if not platform_name or platform_name.upper() == name:
+ if test:
+ return name, test
+
- def getPosixIface():
- from twisted.internet.test import _posixifaces
++def getPlatformAndIfaces(platform_name=None):
+
- log.msg("Attempting to discover network interfaces...")
- ifaces = _posixifaces._interfaces()
- ifup = tryInterfaces(ifaces)
- return ifup
++ client, test = getClientPlatform(platform_name)
+
- def getWindowsIface():
- from twisted.internet.test import _win32ifaces
++ def getPosixIfaces():
++ from twisted.internet.test import _posixifaces
++ all_ifaces = _posixifaces._interfaces()
++ ifup = checkInterfaces(all_ifaces)
++ return all_ifaces
+
- log.msg("Attempting to discover network interfaces...")
- ifaces = _win32ifaces._interfaces()
- ifup = tryInterfaces(ifaces)
- return ifup
++ def getWindowsIfacse():
++ from twisted.internet.test import _win32ifaces
++ all_ifaces = _win32ifaces._interfaces()
++ return all_ifaces
+
- def getPlatformAndIfaces(platform_name=None):
- client, test = getClientPlatform(platform_name)
+ if client:
+ if client == ('LINUX' or 'DARWIN') or client[-3:] == 'BSD':
- return getPosixIface()
++ all_ifaces = getPosixIfaces()
+ elif client == 'WINDOWS':
- return getWindowsIface()
++ all_ifaces = getWindowsIfaces()
+ ## XXX fixme figure out how to get iface for Solaris
+ else:
+ return None
++ ifup = checkInterfaces(all_ifaces)
++ return ifup
+ else:
+ raise UnsupportedPlatform
+
+def checkInterfaces(ifaces=None, timeout=1):
+ """
+ @param ifaces:
+ A dictionary in the form of ifaces['if_name'] = 'if_addr'.
+ """
+ try:
+ from scapy.all import IP, ICMP
+ from scapy.all import sr1 ## we want this check to be blocking
+ except:
+ log.msg(("Scapy required: www.secdev.org/projects/scapy"))
+
+ ifup = {}
+ if not ifaces:
+ log.debug("net.checkInterfaces(): no interfaces specified!")
+ return None
+
+ for iface in ifaces:
+ for ifname, ifaddr in iface:
+ log.debug("net.checkInterfaces(): testing iface {} by pinging"
+ + " local address {}".format(ifname, ifaddr))
+ try:
+ pkt = IP(dst=ifaddr)/ICMP()
+ ans, unans = sr(pkt, iface=ifname, timeout=5, retry=3)
+ except Exception, e:
+ raise PermissionsError if e.find("Errno 1") else log.err(e)
+ else:
+ if ans.summary():
+ log.debug("net.checkInterfaces(): got answer on interface %s"
+ + ":\n%s".format(ifname, ans.summary()))
+ ifup.update(ifname, ifaddr)
+ else:
+ log.debug("Interface test packet was unanswered:\n%s"
+ % unans.summary())
+ if len(ifup) > 0:
+ log.msg("Discovered working network interfaces: %s" % ifup)
+ return ifup
+ else:
+ raise IfaceError
+
+def getNonLoopbackIfaces(platform_name=None):
+ try:
+ ifaces = getPlatformAndIfaces(platform_name)
+ except UnsupportedPlatform, up:
+ log.err(up)
+
+ if not ifaces:
+ log.msg("Unable to discover network interfaces...")
+ return None
+ else:
+ found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo']
+ log.debug("utils.net.getClientIfaces: Found non-loopback interfaces: %s"
- % pprint(found))
++ % found)
+ try:
+ interfaces = checkInterfaces(found)
+ except IfaceError, ie:
+ log.err(ie)
+ return None
+ else:
+ return interfaces
+
+def getNetworksFromRoutes():
- from scapy.all import conf, ltoa
++ from scapy.all import conf, ltoa, read_routes
+ from ipaddr import IPNetwork, IPAddress
+
+ ## Hide the 'no routes' warnings
+ conf.verb = 0
+
+ networks = []
- client = conf.route
- log.debug("Local Routing Table:\n{}".format(client))
-
- for nw, nm, gw, iface, addr in client.routes:
++ for nw, nm, gw, iface, addr in read_routes():
+ n = IPNetwork( ltoa(nw) )
+ (n.netmask, n.gateway, n.ipaddr) = [IPAddress(x) for x in [nm, gw, addr]]
+ n.iface = iface
+ if not n.compressed in networks:
+ networks.append(n)
+
+ return networks
+
+def getDefaultIface():
+ networks = getNetworksFromRoutes()
+ for net in networks:
+ if net.is_private:
- return net
++ return net.iface
+ raise IfaceError
+
+def getLocalAddress():
+ default_iface = getDefaultIface()
+ return default_iface.ipaddr
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits