[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Merge branch 'master' of git-rw.torproject.org:ooni-probe
commit 5c29daa8c9d33f272f618d37d1ba76b5f7998640
Merge: 85f0da8 e10033c
Author: Isis Lovecruft <isis@xxxxxxxxxxxxxx>
Date: Thu Nov 8 06:40:55 2012 +0000
Merge branch 'master' of git-rw.torproject.org:ooni-probe
Conflicts:
ooni/nettest.py
ooni/oonicli.py
ooni/runner.py
.gitignore | 2 +
before_i_commit.sh | 37 ++
data/Makefile | 10 +
docs/source/api/modules.rst | 7 +
docs/source/api/ooni.kit.rst | 19 +
docs/source/api/ooni.lib.rst | 36 ++
docs/source/api/ooni.rst | 15 +-
docs/source/api/ooni.templates.rst | 15 +-
docs/source/api/ooni.utils.rst | 114 ++++-
docs/source/index.rst | 41 +-
docs/source/writing_tests.rst | 3 +-
inputs/Makefile | 3 +
inputs/captive_portal_tests.txt.good | 4 -
inputs/cctld.txt | 511 ------------------
inputs/dns_servers.txt.bak | 6 -
inputs/dns_servers.txt.bak2 | 1 -
inputs/example_exp_list.txt | 3 -
inputs/major_isp_dns_servers.txt | 796 -----------------------------
inputs/short_hostname_list.txt | 7 -
inputs/tld-list-cc.txt | 511 ------------------
inputs/tld-list-mozilla.txt | 5 -
inputs/top-1m.txt.bak2 | 11 -
nettests/bridge_reachability/bridget.py | 462 +++++++++++++++++
nettests/bridge_reachability/echo.py | 195 +++++++
nettests/bridget.py | 499 ------------------
nettests/core/captiveportal.py | 5 +-
nettests/core/dnstamper.py | 97 +++--
nettests/core/echo.py | 1 -
nettests/core/keyword_filtering.py | 39 ++
nettests/core/url_list.py | 23 +
nettests/echo.py | 196 -------
nettests/examples/example_httpt.py | 9 +-
nettests/examples/example_scapyt.py | 1 +
ooni/__init__.py | 15 +-
ooni/config.py | 38 ++-
ooni/custodiet.py | 421 ---------------
ooni/inputunit.py | 133 +-----
ooni/lib/Makefile | 36 --
ooni/lib/txscapy.py | 55 ++-
ooni/nettest.py | 391 +++------------
ooni/nodes.py | 176 +++++++
ooni/oonicli.py | 40 +-
ooni/plugoo/__init__.py | 47 --
ooni/plugoo/assets.py | 62 ---
ooni/plugoo/interface.py | 56 --
ooni/plugoo/nodes.py | 176 -------
ooni/plugoo/reports.py | 145 ------
ooni/plugoo/tests.py | 142 -----
ooni/plugoo/work.py | 148 ------
ooni/protocols/http.py | 141 -----
ooni/protocols/scapyproto.py | 55 --
ooni/reporter.py | 95 ++--
ooni/runner.py | 261 ++++-------
ooni/scaffolding.py | 78 ---
ooni/templates/httpt.py | 51 ++-
ooni/templates/scapyt.py | 108 ++++-
ooni/utils/geodata.py | 31 ++
ooni/utils/legacy.py | 459 -----------------
ooni/utils/log.py | 20 +-
ooni/utils/onion.py | 4 +-
oonib/oonibackend.py | 12 +-
oonib/report/api.py | 63 +--
oonib/report/db/models.py | 83 ---
ooniprobe.conf | 25 +
requirements.txt | 27 +-
test_inputs/README | 48 ++
test_inputs/dns_tamper_file.txt | 3 +
test_inputs/dns_tamper_test_resolvers.txt | 2 +
test_inputs/http_host_file.txt | 2 +
test_inputs/keyword_filtering_file.txt | 2 +
test_inputs/url_lists_file.txt | 2 +
tests/test_dns.py | 24 +
72 files changed, 1868 insertions(+), 5493 deletions(-)
diff --cc nettests/bridge_reachability/echo.py
index 0000000,0c20a3f..e59eee5
mode 000000,100644..100644
--- a/nettests/bridge_reachability/echo.py
+++ b/nettests/bridge_reachability/echo.py
@@@ -1,0 -1,195 +1,195 @@@
+ #!/usr/bin/env python
+ # -*- coding: utf-8 -*-
+ #
+ # +---------+
+ # | echo.py |
+ # +---------+
+ # A simple ICMP-8 ping test.
+ #
+ # :author: Isis Lovecruft
+ # :version: 0.0.1-pre-alpha
+ # :license: (c) 2012 Isis Lovecruft
+ # see attached LICENCE file
+ #
+
+ import os
+ import sys
+
+ from pprint import pprint
+
+ from twisted.internet import reactor
+ from twisted.plugin import IPlugin
+ from twisted.python import usage
+ from ooni.nettest import NetTestCase
+ from ooni.utils import log, Storage
+ from ooni.utils.net import PermissionsError, IfaceError
+
+ try:
+ from scapy.all import sr1, IP, ICMP ## XXX v4/v6?
+ from ooni.lib import txscapy
+ from ooni.lib.txscapy import txsr, txsend
+ from ooni.templates.scapyt import ScapyTest
+ except:
+ log.msg("This test requires scapy, see www.secdev.org/projects/scapy")
+
+ ## xxx TODO: move these to a utility function for determining OSes
+ LINUX=sys.platform.startswith("linux")
+ OPENBSD=sys.platform.startswith("openbsd")
+ FREEBSD=sys.platform.startswith("freebsd")
+ NETBSD=sys.platform.startswith("netbsd")
+ DARWIN=sys.platform.startswith("darwin")
+ SOLARIS=sys.platform.startswith("sunos")
+ WINDOWS=sys.platform.startswith("win32")
+
+ class EchoTest(ScapyTest):
+ """
+ xxx fill me in
+ """
+ name = 'echo'
+ author = 'Isis Lovecruft <isis@xxxxxxxxxxxxxx>'
+ description = 'A simple ICMP-8 test to see if a host is reachable.'
+ version = '0.0.1'
+ inputFile = ['file', 'f', None, 'File of list of IPs to ping']
+ requiresRoot = True
+
+ optParameters = [
+ ['interface', 'i', None, 'Network interface to use'],
+ ['count', 'c', 5, 'Number of packets to send', int],
+ ['size', 's', 56, 'Number of bytes to send in ICMP data field', int],
+ ['ttl', 'l', 25, 'Set the IP Time to Live', int],
+ ['timeout', 't', 2, 'Seconds until timeout if no response', int],
+ ['pcap', 'p', None, 'Save pcap to this file'],
+ ['receive', 'r', True, 'Receive response packets']
+ ]
+
+ def setUp(self, *a, **kw):
+ '''
+ :ivar ifaces:
+ Struct returned from getifaddrs(3) and turned into a tuple in the
+ form (*ifa_name, AF_FAMILY, *ifa_addr)
+ '''
+
+ if self.localOptions:
+ log.debug("%s: local_options found" % self.name)
+ for key, value in self.localOptions.items():
+ log.debug("setting self.%s = %s" % (key, value))
+ setattr(self, key, value)
+
+ ## xxx is this now .subOptions?
+ #self.inputFile = self.localOptions['file']
+ self.timeout *= 1000 ## convert to milliseconds
+
+ if not self.interface:
+ log.msg("No network interface specified!")
+ log.debug("OS detected: %s" % sys.platform)
+ if LINUX or OPENBSD or NETBSD or FREEBSD or DARWIN or SOLARIS:
+ from twisted.internet.test import _posixifaces
+ log.msg("Attempting to discover network interfaces...")
+ ifaces = _posixifaces._interfaces()
+ elif WINDOWS:
+ from twisted.internet.test import _win32ifaces
+ log.msg("Attempting to discover network interfaces...")
+ ifaces = _win32ifaces._interfaces()
+ else:
+ log.debug("Client OS %s not accounted for!" % sys.platform)
+ log.debug("Unable to discover network interfaces...")
+ ifaces = [('lo', '')]
+
+ ## found = {'eth0': '1.1.1.1'}
+ found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo']
+ log.info("Found interfaces:\n%s" % pprint(found))
+ self.interfaces = self.tryInterfaces(found)
+ else:
+ ## xxx need a way to check that iface exists, is up, and
+ ## we have permissions on it
+ log.debug("Our interface has been set to %s" % self.interface)
+
+ if self.pcap:
+ try:
+ self.pcapfile = open(self.pcap, 'a+')
+ except:
+ log.msg("Unable to write to pcap file %s" % self.pcap)
+ self.pcapfile = None
+
+ try:
+ assert os.path.isfile(self.file)
+ fp = open(self.file, 'r')
+ except Exception, e:
+ hosts = ['8.8.8.8', '38.229.72.14']
+ log.err(e)
+ else:
+ self.inputs = self.inputProcessor(fp)
+ self.removePorts(hosts)
+
+ log.debug("Initialization of %s test completed with:\n%s"
+ % (self.name, ''.join(self.__dict__)))
+
+ @staticmethod
- def inputParser(inputs):
++ def inputParser(self, one_input):
+ log.debug("Removing possible ports from host addresses...")
+ log.debug("Initial inputs:\n%s" % pprint(inputs))
+
- assert isinstance(inputs, list)
- hosts = [h.rsplit(':', 1)[0] for h in inputs]
++ #host = [h.rsplit(':', 1)[0] for h in inputs]
++ host = h.rsplit(':', 1)[0]
+ log.debug("Inputs converted to:\n%s" % hosts)
+
- return hosts
++ return host
+
+ def tryInterfaces(self, ifaces):
+ try:
+ from scapy.all import sr1 ## we want this check to be blocking
+ except:
+ log.msg("This test requires scapy: www.secdev.org/projects/scapy")
+ raise SystemExit
+
+ ifup = {}
+ while ifaces:
+ for ifname, ifaddr in ifaces:
+ log.debug("Currently testing network capabilities of interface"
+ + "%s by sending a packet to our address %s"
+ % (ifname, ifaddr))
+ try:
+ pkt = IP(dst=ifaddr)/ICMP()
+ ans, unans = sr(pkt, iface=ifname, timeout=self.timeout)
+ except Exception, e:
+ raise PermissionsError if e.find("Errno 1") else log.err(e)
+ else:
+ ## xxx i think this logic might be wrong
+ log.debug("Interface test packet\n%s\n\n%s"
+ % (pkt.summary(), pkt.show2()))
+ if ans.summary():
+ log.info("Received answer for test packet on interface"
+ +"%s :\n%s" % (ifname, ans.summary()))
+ ifup.update(ifname, ifaddr)
+ else:
+ log.info("Our interface test packet was unanswered:\n%s"
+ % unans.summary())
+
+ if len(ifup) > 0:
+ log.msg("Discovered the following working network interfaces: %s"
+ % ifup)
+ return ifup
+ else:
+ raise IfaceError("Could not find a working network interface.")
+
+ def buildPackets(self):
+ log.debug("self.input is %s" % self.input)
+ log.debug("self.hosts is %s" % self.hosts)
+ for addr in self.input:
+ packet = IP(dst=self.input)/ICMP()
+ self.request.append(packet)
+ return packet
+
+ def test_icmp(self):
+ if self.recieve:
+ self.buildPackets()
+ all = []
+ for packet in self.request:
+ d = self.sendReceivePackets(packets=packet)
+ all.append(d)
+ self.response.update({packet: d})
+ d_list = defer.DeferredList(all)
+ return d_list
+ else:
+ d = self.sendPackets()
+ return d
diff --cc ooni/nettest.py
index 84c3289,3534a32..27e733c
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@@ -1,17 -1,8 +1,16 @@@
# -*- encoding: utf-8 -*-
#
-# :authors: Arturo "hellais" Filastò <art@xxxxxxxxx>
+# nettest.py
+# ------------------->
+#
+# :authors: Arturo "hellais" Filastò <art@xxxxxxxxx>,
+# Isis Lovecruft <isis@xxxxxxxxxxxxxx>
# :licence: see LICENSE
+# :copyright: 2012 Arturo Filasto, Isis Lovecruft
+# :version: 0.1.0-alpha
+#
+# <-------------------
- from functools import partial
import sys
import os
import itertools
@@@ -392,10 -118,10 +127,10 @@@ class NetTestCase(unittest.TestCase)
"""
name = "I Did Not Change The Name"
author = "Jane Doe <foo@xxxxxxxxxxx>"
- version = "0"
+ version = "0.0.0"
+ inputs = [None]
inputFile = None
- inputs = [None]
report = {}
report['errors'] = []
diff --cc ooni/oonicli.py
index a61b3f9,5ee6deb..4f5eac5
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@@ -41,21 -41,19 +41,15 @@@ class Options(usage.Options, app.Reacto
optFlags = [["help", "h"],
['debug-stacktraces', 'B',
-- 'Report deferred creation and callback stack traces'],
-- ]
++ 'Report deferred creation and callback stack traces'],]
-- optParameters = [
-- ["reportfile", "o", None, "report file name"],
- ["logfile", "l", "test.log", "log file name"],
- ['temp-directory', None, '_ooni_temp',
- 'Path to use as working directory for tests.']
- ]
- ["logfile", "l", None, "log file name"],
- ]
++ optParameters = [["reportfile", "o", None, "report file name"],
++ ["logfile", "l", None, "log file name"],]
compData = usage.Completions(
extraActions=[usage.CompleteFiles(
"*.py", descr="file | module | package | TestCase | testMethod",
-- repeat=True)],
-- )
++ repeat=True)],)
tracer = None
@@@ -86,29 -85,22 +80,27 @@@
def run():
- if len(sys.argv) == 1:
- sys.argv.append("--help")
+ """
+ Call me to begin testing a file or module.
+ """
-
- config = Options()
-
+ cmd_line_options = Options()
+ if len(sys.argv) == 1:
- config.getUsage()
-
++ cmd_line_options.getUsage()
try:
- config.parseOptions()
+ cmd_line_options.parseOptions()
- except usage.error, ue:
+ except usage.UsageError, ue:
raise SystemExit, "%s: %s" % (sys.argv[0], ue)
+ log.start()
- log.debug("oonicli.run: config set to %s" % config)
+
- if config['debug-stacktraces']:
+ if cmd_line_options['debug-stacktraces']:
defer.setDebugging(True)
- classes = runner.findTestClassesFromConfig(config)
- casesList, options = runner.loadTestsAndOptions(classes, config)
+ classes = runner.findTestClassesFromConfig(cmd_line_options)
+ casesList, options = runner.loadTestsAndOptions(classes, cmd_line_options)
for idx, cases in enumerate(casesList):
- orunner = runner.ORunner(cases, options[idx], config)
+ orunner = runner.ORunner(cases, options[idx], cmd_line_options)
+ log.start(cmd_line_options['logfile'])
orunner.run()
+
diff --cc ooni/runner.py
index 52c6596,797a9d4..c8c6812
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@@ -239,27 -149,31 +149,34 @@@ class ORunner(object)
self.cases = cases
self.options = options
+ log.debug("ORunner: cases=%s" % type(cases))
+ log.debug("ORunner: options=%s" % options)
+
try:
- first = options.pop(0)
- except:
- first = options
-
- if 'inputs' in first:
- self.inputs = self.options['inputs']
+ assert len(options) != 0, "Length of options is zero!"
+ except AssertionError, ae:
+ self.inputs = []
+ log.err(ae)
else:
- log.msg("Could not find inputs!")
- self.inputs = [None]
+ try:
+ first = options.pop(0)
+ except:
+ first = options
+
+ if 'inputs' in first:
+ self.inputs = options['inputs']
+ else:
+ log.msg("Could not find inputs!")
+ log.msg("options[0] = %s" % first)
+ self.inputs = [None]
try:
- reportFile = open(config['reportfile'], 'a+')
- except:
+ reportFile = open(cmd_line_options['reportfile'], 'a+')
+ except TypeError:
filename = 'report_'+date.timestamp()+'.yaml'
reportFile = open(filename, 'a+')
- self.reporterFactory = ReporterFactory(
- reportFile, testSuite=self.baseSuite(self.cases))
+ self.reporterFactory = ReporterFactory(reportFile,
+ testSuite=self.baseSuite(self.cases))
def runWithInputUnit(self, input_unit):
idx = 0
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits