[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Merge branch 'master' into mastermerge



commit c7145bb3177eefcfcebb013db379f34b4c8baf35
Merge: a319db3 abfcf7c
Author: Arturo Filastò <hellais@xxxxxxxxxxxxxx>
Date:   Thu May 31 04:44:39 2012 +0200

    Merge branch 'master' into mastermerge
    
    Conflicts:
    	ooni/plugoo/reports.py

 HACKING                     |    2 +-
 LICENSE                     |   26 +++++++++
 TODO                        |    5 ++
 ooni/ooni-probe.conf        |    7 ++-
 ooni/oonitests/dnstamper.py |  122 ++++++++++++++++++++++++++++++++++++------
 ooni/oonitests/httphost.py  |    5 ++-
 6 files changed, 146 insertions(+), 21 deletions(-)

diff --cc ooni/ooni-probe.conf
index 336af99,0000000..d95e410
mode 100644,000000..100644
--- a/ooni/ooni-probe.conf
+++ b/ooni/ooni-probe.conf
@@@ -1,72 -1,0 +1,77 @@@
 +# ooni-probe
 +#
 +# These are the global configuration parameters necessary to
 +# make ooni-probe work
 +[main]
 +reportdir = reports/
 +logfile = ooniprobe.log
 +assetdir = assets/
 +testdir = oonitests/
 +
 +loglevel = DEBUG
 +consoleloglevel = DEBUG
 +proxyaddress = 127.0.0.1:9050
 +
 +# The following configurations are for searching for PlanetLab
 +# nodes, adding them to a slice, and PlanetLab general API
 +# authentication:
 +pl_username = yourusername
 +pl_password = yourpassword
 +
 +# These are configurations specific to the tests that should be
 +# run by ooni-probe
 +[tests]
 +run = dnstamper
 +### DNS testing related config parameters
 +
 +# This is the list of hostnames that must be looked up
 +dns_experiment = top-1m.txt
 +
 +# This is the dns servers to be tested
 +dns_experiment_dns = dns_servers.txt
 +
 +# This is the control known good DNS server
- dns_control_server = 8.8.8.8
++dns_control_server = 91.191.136.152
++
++# Specify whether the dnstamper test should attempt to remove 
++# GeoIP-based false positives by doing a reverse DNS resolve
++# on positive results.
++dns_reverse_lookup = true
 +
 +### traceroute testing related config parameters
 +
 +# This is the list of ips to traceroute to
 +traceroute = example_exp_list.txt
 +
 +# This is the list of ports that should be used
 +# src_x,src_y,src_z|dst_x,dst_y,dst_z
 +traceroute_ports = 0,53,80,123,443|0,53,80,123,443
 +
 +# The protocol to be used in the scan
 +traceroute_proto = UDP, TCP, ICMP
 +
 +### keyword injection related tests
 +
 +# List of keywords
 +keywords = keywordlist.txt
 +
 +# hosts
 +keywords_hosts = hostslist.txt
 +
 +# Methods to be used for testing
 +keyword_method = http,telnet
 +
 +### Tor bridge testing
 +
 +tor_bridges = bridgetests.txt
 +tor_bridges_timeout = 40
 +
 +[report]
 +file = report.log
 +timestamp = true
 +#ssh = 127.0.0.1:22
 +#ssh_user = theusername
 +#ssh_password = thepassword
 +#ssh_keyfile = ~/.ssh/mykey_rsa
 +#ssh_rpath = ~/ooni-probe/
 +#tcp = "127.0.0.1:9088"
diff --cc ooni/oonitests/dnstamper.py
index 68be12d,0000000..498ba04
mode 100644,000000..100644
--- a/ooni/oonitests/dnstamper.py
+++ b/ooni/oonitests/dnstamper.py
@@@ -1,70 -1,0 +1,156 @@@
++# -*- coding: utf-8 -*-
++"""
++    dnstamper
++    *********
++
++    This test resolves DNS for a list of domain names, one per line, in the
++    file specified in the ooni-config under the setting "dns_experiment". If
++    the file is top-1m.txt, the test will be run using Amazon's list of top
++    one million domains. The experimental dns servers to query should
++    be specified one per line in assets/dns_servers.txt.
++
++    The test reports censorship if the cardinality of the intersection of
++    the query result set from the control server and the query result set
++    from the experimental server is zero, which is to say, if the two sets
++    have no matching results whatsoever.
++
++    NOTE: This test frequently results in false positives due to GeoIP-based
++    load balancing on major global sites such as google, facebook, and
++    youtube, etc.
++
++    :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft
++    :license: see LICENSE for more details
++"""
++
++try:
++    from dns import resolver, reversename
++except:
++    print "Error: dnspython is not installed! (http://www.dnspython.org/)"
 +try:
-     from dns import resolver
++    import gevent
 +except:
-     print "Error dnspython is not installed! (http://www.dnspython.org/)"
- import gevent
++    print "Error: gevent is not installed! (http://www.gevent.org/)"
++
 +import os
++
 +import plugoo
 +from plugoo.assets import Asset
 +from plugoo.tests import Test
 +
- 
 +__plugoo__ = "DNST"
 +__desc__ = "DNS censorship detection test"
 +
++class Top1MAsset(Asset):
++    """
++    Class for parsing top-1m.txt as an asset.
++    """
++    def __init__(self, file=None):
++        self = Asset.__init__(self, file)
++
++    def parse_line(self, line):
++        self = Asset.parse_line(self, line)
++        return line.split(',')[1].replace('\n','')
++
 +class DNSTAsset(Asset):
++    """
++    Creates DNS testing specific Assets.
++    """
 +    def __init__(self, file=None):
-         self = asset.__init__(self, file)
++        self = Asset.__init__(self, file)
 +
 +class DNST(Test):
 +    def lookup(self, hostname, ns):
++        """
++        Resolves a hostname through a DNS nameserver, ns, to the corresponding
++        IP address(es).
++        """
 +        res = resolver.Resolver(configure=False)
 +        res.nameservers = [ns]
 +        answer = res.query(hostname)
 +
 +        ret = []
 +
 +        for data in answer:
 +            ret.append(data.address)
 +
 +        return ret
 +
++    def reverse_lookup(self, ip, ns):
++        """
++        Attempt to do a reverse DNS lookup to determine if the control and exp
++        sets from a positive result resolve to the same domain, in order to
++        remove false positives due to GeoIP load balancing.
++        """
++        res = resolver.Resolver(configure=False)
++        res.nameservers = [ns]
++        n = reversename.from_address(ip)
++        revn = res.query(n, "PTR").__iter__().next().to_text()[:-1]
++
++        return revn
++
 +    def experiment(self, *a, **kw):
++        """
++        Compares the lookup() sets of the control and experiment groups.
++        """
 +        # this is just a dirty hack
 +        address = kw['data'][0]
 +        ns = kw['data'][1]
 +
 +        config = self.config
++        ctrl_ns = config.tests.dns_control_server
 +
 +        print "ADDRESS: %s" % address
 +        print "NAMESERVER: %s" % ns
 +
 +        exp = self.lookup(address, ns)
-         control = self.lookup(address, config.tests.dns_control_server)
++        control = self.lookup(address, ctrl_ns)
++
++        result = []
 +
 +        if len(set(exp) & set(control)) > 0:
-             print "%s : no tampering on %s" % (address, ns)
-             return (address, ns, False)
++            print "Address %s has not tampered with on DNS server %s\n" % (address, ns)
++            result = (address, ns, exp, control, False)
++            return result
 +        else:
-             print "%s : possible tampering on %s (%s, %s)" % (address, ns, exp, control)
-             return (address, ns, exp, control, True)
++            print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control)
++            result = (address, ns, exp, control, True)
++
++            if config.tests.dns_reverse_lookup:
++
++                exprevn = [self.reverse_lookup(ip, ns) for ip in exp]
++                ctrlrevn = [self.reverse_lookup(ip, ctrl_ns)
++                            for ip in control]
++
++                if len(set(exprevn) & set(ctrlrevn)) > 0:
++                    print "Further testing has eliminated this as a false positive."
++                else:
++                    print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn)
++                return result
++
++            else:
++                print "\n"
++                return result
 +
 +def run(ooni):
-     """Run the test
++    """
++    Run the test.
 +    """
 +    config = ooni.config
 +    urls = []
 +
-     dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, \
-                                             config.tests.dns_experiment))
-     dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, \
++    if (config.tests.dns_experiment == "top-1m.txt"):
++        dns_experiment = Top1MAsset(os.path.join(config.main.assetdir,
++                                                 config.tests.dns_experiment))
++    else:
++        dns_experiment = DNSTAsset(os.path.join(config.main.assetdir,
++                                                config.tests.dns_experiment))
++    dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir,
 +                                                config.tests.dns_experiment_dns))
 +
 +    assets = [dns_experiment, dns_experiment_dns]
 +
 +    dnstest = DNST(ooni)
-     ooni.logger.info("starting test")
-     dnstest.run(assets)
-     ooni.logger.info("finished")
- 
++    ooni.logger.info("Beginning dnstamper test...")
++    dnstest.run(assets, {'index': 1})
++    ooni.logger.info("Dnstamper test completed!")
 +
diff --cc ooni/oonitests/httphost.py
index 6446e1f,0000000..25adff2
mode 100644,000000..100644
--- a/ooni/oonitests/httphost.py
+++ b/ooni/oonitests/httphost.py
@@@ -1,132 -1,0 +1,135 @@@
 +"""
 +    HTTP Host based filtering
 +    *************************
 +
 +    This test detect HTTP Host field
 +    based filtering.
 +    It is used to detect censorship on
 +    performed with Web Guard (used by
 +    T-Mobile US).
 +"""
 +import os
 +from datetime import datetime
 +from gevent import monkey
 +
 +import urllib2
 +import httplib
 +# WARNING! Using gevent's socket
 +# introduces the 0x20 DNS "feature".
 +# This will result is weird DNS requests
 +# appearing on the wire.
 +monkey.patch_socket()
 +
- from BeautifulSoup import BeautifulSoup
++try:
++    from BeautifulSoup import BeautifulSoup
++except:
++    print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/";
 +
 +from plugoo.assets import Asset
 +from plugoo.tests import Test
 +
 +__plugoo__ = "HTTP Host"
 +__desc__ = "This detects HTTP Host field based filtering"
 +
 +class HTTPHostAsset(Asset):
 +    """
 +    This is the asset that should be used by the Test. It will
 +    contain all the code responsible for parsing the asset file
 +    and should be passed on instantiation to the test.
 +    """
 +    def __init__(self, file=None):
 +        self = Asset.__init__(self, file)
 +
 +    def parse_line(self, line):
 +        return line.split(',')[1].replace('\n','')
 +
 +class HTTPHost(Test):
 +    """
 +    The main Test class
 +    """
 +
 +    def check_response(self, response):
 +        soup = BeautifulSoup(response)
 +        if soup.head.title.string == "Content Filtered":
 +            # Response indicates censorship
 +            return True
 +        else:
 +            # Response does not indicate censorship
 +            return False
 +
 +
 +    def is_censored(self, response):
 +        if response:
 +            soup = BeautifulSoup(response)
 +            censored = self.check_response(response)
 +        else:
 +            censored = "unreachable"
 +        return censored
 +
 +    def urllib2_test(self, control_server, host):
 +        req = urllib2.Request(control_server)
 +        req.add_header('Host', host)
 +        try:
 +            r = urllib2.urlopen(req)
 +            response = r.read()
 +            censored = self.is_censored(response)
 +        except Exception, e:
 +            censored = "Error! %s" % e
 +
 +        return censored
 +
 +    def httplib_test(self, control_server, host):
 +        try:
 +            conn = httplib.HTTPConnection(control_server)
 +            conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True)
 +            conn.putheader("Host", host)
 +            conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6")
 +            conn.endheaders()
 +            r = conn.getresponse()
 +            response = r.read()
 +            censored = self.is_censored(response)
 +        except Exception, e:
 +            censored = "Error! %s" % e
 +
 +        return censored
 +
 +
 +    def experiment(self, *a, **kw):
 +        """
 +        Try to connect to the control server with
 +        the specified host field.
 +        """
 +        host = kw['data']
 +        control_server = kw['control_server']
 +        self.logger.info("Testing %s (%s)" % (host, control_server))
 +
 +        #censored = self.urllib2_test(control_server, host)
 +        censored = self.httplib_test(control_server, host)
 +
 +        self.logger.info("%s: %s" % (host, censored))
 +        return {'Time': datetime.now(),
 +                'Host': host,
 +                'Censored': censored}
 +
 +
 +def run(ooni):
 +    """
 +    This is the function that will be called by OONI
 +    and it is responsible for instantiating and passing
 +    the arguments to the Test class.
 +    """
 +    config = ooni.config
 +
 +    # This the assets array to be passed to the run function of
 +    # the test
 +    assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \
 +                                            "top-1m.csv"))]
 +
 +    # Instantiate the Test
 +    thetest = HTTPHost(ooni)
 +    ooni.logger.info("starting HTTP Host Test...")
 +    # Run the test with argument assets
 +    thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'})
 +    ooni.logger.info("finished.")
 +
 +



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits