[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Merge branch 'twisted' of github.com:hellais/ooni-probe



commit baab66b10eccb5a3bc40addc84de0e45f2897497
Merge: abfcf7c 59f1b4e
Author: Arturo Filastò <hellais@xxxxxxxxxxxxxx>
Date:   Tue May 29 03:23:01 2012 +0200

    Merge branch 'twisted' of github.com:hellais/ooni-probe

 HACKING                                            |   36 +-
 assets/example.txt                                 |    2 -
 assets/tcpscan.txt                                 |    1 -
 config.py                                          |   53 -
 docs/writing_tests.txt                             |   12 +
 install.txt                                        |    4 -
 lib/traceroute.py                                  |   32 -
 lib/txtraceroute.py                                |  389 -
 logo.py                                            |  214 -
 nodes.conf                                         |   53 -
 old-to-be-ported-code/AUTHORS                      |    3 +
 old-to-be-ported-code/CHANGES.yaml                 |    5 +
 old-to-be-ported-code/HACKING                      |   21 +
 old-to-be-ported-code/INSTALL                      |    8 +
 old-to-be-ported-code/LICENSE                      |   28 +
 old-to-be-ported-code/MANIFEST.in                  |    6 +
 old-to-be-ported-code/Makefile                     |  182 +
 old-to-be-ported-code/README.API                   |   29 +
 old-to-be-ported-code/README.examples              |   10 +
 old-to-be-ported-code/README.rst                   |   40 +
 old-to-be-ported-code/RFA.md                       |  101 +
 old-to-be-ported-code/TODO                         |  418 +
 old-to-be-ported-code/TODO.plgoons                 |   79 +
 old-to-be-ported-code/bin/ooni-probe               |   10 +
 old-to-be-ported-code/keyword-lists/Makefile       |    3 +
 old-to-be-ported-code/keyword-lists/keyfile        |  256 +
 old-to-be-ported-code/list/list.txt                |    2 +
 old-to-be-ported-code/ooni-probe.diff              |  358 +
 old-to-be-ported-code/ooni/.DS_Store               |  Bin 0 -> 15364 bytes
 old-to-be-ported-code/ooni/__init__.py             |   12 +
 old-to-be-ported-code/ooni/captive_portal.py       |   62 +
 old-to-be-ported-code/ooni/command.py              |  250 +
 old-to-be-ported-code/ooni/common.py               |  139 +
 old-to-be-ported-code/ooni/dns_cc_check.py         |   48 +
 old-to-be-ported-code/ooni/dns_poisoning.py        |   43 +
 old-to-be-ported-code/ooni/dnsooni.py              |  356 +
 old-to-be-ported-code/ooni/empty.txt               |    9 +
 old-to-be-ported-code/ooni/helpers.py              |   38 +
 old-to-be-ported-code/ooni/http.py                 |  306 +
 old-to-be-ported-code/ooni/input.py                |   33 +
 old-to-be-ported-code/ooni/namecheck.py            |   39 +
 old-to-be-ported-code/ooni/output.py               |   21 +
 .../ooni/plugins/captiveportal_plgoo.py            |   55 +
 .../ooni/plugins/dnstest_plgoo.py                  |   84 +
 old-to-be-ported-code/ooni/plugins/http_plgoo.py   |   70 +
 old-to-be-ported-code/ooni/plugins/marco_plgoo.py  |  377 +
 .../ooni/plugins/netalyzr_plgoo.py                 |   30 +
 .../ooni/plugins/old_stuff_to_convert/README       |   47 +
 .../plugins/old_stuff_to_convert/README.automation |   11 +
 .../ooni/plugins/old_stuff_to_convert/TODO         |    6 +
 .../plugins/old_stuff_to_convert/cached-consensus  |11724 ++++++++++++++++++++
 .../plugins/old_stuff_to_convert/connectback.sh    |   56 +
 .../plugins/old_stuff_to_convert/dirconntest.sh    |   52 +
 .../plugins/old_stuff_to_convert/dns-checker.sh    |    7 +
 .../old_stuff_to_convert/generic-host-test.sh      |   33 +
 .../ooni/plugins/old_stuff_to_convert/host-prep.sh |   20 +
 .../plugins/old_stuff_to_convert/install-probe.sh  |   27 +
 .../ooni/plugins/old_stuff_to_convert/run-tests.sh |   11 +
 .../plugins/old_stuff_to_convert/twitter-test.sh   |   33 +
 old-to-be-ported-code/ooni/plugins/proxy_plgoo.py  |   69 +
 .../ooni/plugins/simple_dns_plgoo.py               |   35 +
 old-to-be-ported-code/ooni/plugins/skel_plgoo.py   |   17 +
 old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml |   33 +
 old-to-be-ported-code/ooni/plugins/tcpcon_plgoo.py |  278 +
 old-to-be-ported-code/ooni/plugins/tor.py          |   80 +
 old-to-be-ported-code/ooni/plugins/torrc           |    9 +
 old-to-be-ported-code/ooni/plugooni.py             |  106 +
 old-to-be-ported-code/ooni/report.py               |   22 +
 old-to-be-ported-code/ooni/transparenthttp.py      |   41 +
 old-to-be-ported-code/ooni/yamlooni.py             |   40 +
 old-to-be-ported-code/proxy-lists/Makefile         |   11 +
 old-to-be-ported-code/proxy-lists/README           |    9 +
 old-to-be-ported-code/proxy-lists/italy-dns.txt    |   46 +
 .../proxy-lists/italy-http-block-pages-notes.txt   |   62 +
 .../proxy-lists/italy-http-block-pages.txt         |    5 +
 .../proxy-lists/parse-trusted-xff.sh               |   16 +
 old-to-be-ported-code/proxy-lists/trusted-xff.html | 7789 +++++++++++++
 old-to-be-ported-code/reports/marco-1.yamlooni     |   86 +
 old-to-be-ported-code/reports/marco-2.yamlooni     |   60 +
 old-to-be-ported-code/reports/marco.log            | 4672 ++++++++
 old-to-be-ported-code/reports/marco.yamlooni       |  241 +
 old-to-be-ported-code/reports/marco_certs-1-2.out  |  307 +
 old-to-be-ported-code/reports/marco_certs-1.out    | 5850 ++++++++++
 old-to-be-ported-code/reports/marco_certs-2.out    | 1182 ++
 old-to-be-ported-code/reports/marco_certs.out      | 1900 ++++
 old-to-be-ported-code/setup.py                     |   10 +
 old-to-be-ported-code/spec/proxooni-spec.txt       |   65 +
 old-to-be-ported-code/test-lists/Makefile          |    7 +
 old-to-be-ported-code/tests/test_import.py         |   10 +
 old-to-be-ported-code/third-party/Makefile         |    3 +
 old-to-be-ported-code/third-party/README           |   14 +
 old/AUTHORS                                        |    3 -
 old/CHANGES.yaml                                   |    5 -
 old/HACKING                                        |   21 -
 old/INSTALL                                        |    8 -
 old/LICENSE                                        |   28 -
 old/MANIFEST.in                                    |    6 -
 old/Makefile                                       |  182 -
 old/README.API                                     |   29 -
 old/README.examples                                |   10 -
 old/README.rst                                     |   40 -
 old/RFA.md                                         |  101 -
 old/TODO                                           |  418 -
 old/TODO.plgoons                                   |   79 -
 old/bin/ooni-probe                                 |   10 -
 old/keyword-lists/Makefile                         |    3 -
 old/keyword-lists/keyfile                          |  256 -
 old/list/list.txt                                  |    2 -
 old/ooni-probe.diff                                |  358 -
 old/ooni/.DS_Store                                 |  Bin 15364 -> 0 bytes
 old/ooni/__init__.py                               |   12 -
 old/ooni/captive_portal.py                         |   62 -
 old/ooni/command.py                                |  250 -
 old/ooni/common.py                                 |  139 -
 old/ooni/dns_cc_check.py                           |   48 -
 old/ooni/dns_poisoning.py                          |   43 -
 old/ooni/dnsooni.py                                |  356 -
 old/ooni/empty.txt                                 |    9 -
 old/ooni/helpers.py                                |   38 -
 old/ooni/http.py                                   |  306 -
 old/ooni/input.py                                  |   33 -
 old/ooni/namecheck.py                              |   39 -
 old/ooni/output.py                                 |   21 -
 old/ooni/plugins/captiveportal_plgoo.py            |   55 -
 old/ooni/plugins/dnstest_plgoo.py                  |   84 -
 old/ooni/plugins/http_plgoo.py                     |   70 -
 old/ooni/plugins/marco_plgoo.py                    |  377 -
 old/ooni/plugins/netalyzr_plgoo.py                 |   30 -
 old/ooni/plugins/old_stuff_to_convert/README       |   47 -
 .../plugins/old_stuff_to_convert/README.automation |   11 -
 old/ooni/plugins/old_stuff_to_convert/TODO         |    6 -
 .../plugins/old_stuff_to_convert/cached-consensus  |11724 --------------------
 .../plugins/old_stuff_to_convert/connectback.sh    |   56 -
 .../plugins/old_stuff_to_convert/dirconntest.sh    |   52 -
 .../plugins/old_stuff_to_convert/dns-checker.sh    |    7 -
 .../old_stuff_to_convert/generic-host-test.sh      |   33 -
 old/ooni/plugins/old_stuff_to_convert/host-prep.sh |   20 -
 .../plugins/old_stuff_to_convert/install-probe.sh  |   27 -
 old/ooni/plugins/old_stuff_to_convert/run-tests.sh |   11 -
 .../plugins/old_stuff_to_convert/twitter-test.sh   |   33 -
 old/ooni/plugins/proxy_plgoo.py                    |   69 -
 old/ooni/plugins/simple_dns_plgoo.py               |   35 -
 old/ooni/plugins/skel_plgoo.py                     |   17 -
 old/ooni/plugins/skel_plgoo.yaml                   |   33 -
 old/ooni/plugins/tcpcon_plgoo.py                   |  278 -
 old/ooni/plugins/tor.py                            |   80 -
 old/ooni/plugins/torrc                             |    9 -
 old/ooni/plugooni.py                               |  106 -
 old/ooni/report.py                                 |   22 -
 old/ooni/transparenthttp.py                        |   41 -
 old/ooni/yamlooni.py                               |   40 -
 old/proxy-lists/Makefile                           |   11 -
 old/proxy-lists/README                             |    9 -
 old/proxy-lists/italy-dns.txt                      |   46 -
 old/proxy-lists/italy-http-block-pages-notes.txt   |   62 -
 old/proxy-lists/italy-http-block-pages.txt         |    5 -
 old/proxy-lists/parse-trusted-xff.sh               |   16 -
 old/proxy-lists/trusted-xff.html                   | 7789 -------------
 old/reports/marco-1.yamlooni                       |   86 -
 old/reports/marco-2.yamlooni                       |   60 -
 old/reports/marco.log                              | 4672 --------
 old/reports/marco.yamlooni                         |  241 -
 old/reports/marco_certs-1-2.out                    |  307 -
 old/reports/marco_certs-1.out                      | 5850 ----------
 old/reports/marco_certs-2.out                      | 1182 --
 old/reports/marco_certs.out                        | 1900 ----
 old/setup.py                                       |   10 -
 old/spec/proxooni-spec.txt                         |   65 -
 old/test-lists/Makefile                            |    7 -
 old/tests/test_import.py                           |   10 -
 old/third-party/Makefile                           |    3 -
 old/third-party/README                             |   14 -
 ooni-probe.conf                                    |   77 -
 ooni/assets/bridgetests.txt                        |    3 +
 ooni/assets/example.txt                            |    2 +
 ooni/assets/tcpscan.txt                            |    1 +
 ooni/config.py                                     |   53 +
 ooni/lib/Makefile                                  |   14 +
 ooni/logo.py                                       |  214 +
 ooni/nodes.conf                                    |   53 +
 ooni/ooni-probe.conf                               |   77 +
 ooni/oonicli.py                                    |  139 +
 ooni/ooniprobe.log                                 |    2 +
 ooni/ooniprobe.py                                  |  191 +
 ooni/oonitests/bridget.py                          |  373 +
 ooni/oonitests/dnstamper.py                        |  156 +
 ooni/oonitests/httphost.py                         |  135 +
 ooni/oonitests/squid.py                            |  113 +
 ooni/oonitests/tcpscan.py                          |   86 +
 ooni/oonitests/template.py                         |   65 +
 ooni/oonitests/traceroute.py                       |  110 +
 ooni/plugins/.bridget.py.swp                       |  Bin 0 -> 12288 bytes
 ooni/plugins/.dnstamper.py.swp                     |  Bin 0 -> 12288 bytes
 ooni/plugins/__init__.py                           |    3 +
 ooni/plugins/bridget.py                            |   36 +
 ooni/plugins/dnstamper.py                          |   22 +
 ooni/plugins/dropin.cache                          |   76 +
 ooni/plugins/skel.py                               |   21 +
 ooni/plugoo/__init__.py                            |   80 +
 ooni/plugoo/assets.py                              |   56 +
 ooni/plugoo/interface.py                           |    2 +
 ooni/plugoo/nodes.py                               |  180 +
 ooni/plugoo/reports.py                             |  175 +
 ooni/plugoo/tests.py                               |  229 +
 ooni/plugoo/work.py                                |  162 +
 ooni/skel.py                                       |   10 +
 ooni/tests/worker_test.py                          |   27 +
 ooni/utils.py                                      |  146 +
 oonib/dnsbackend.py                                |   16 +
 oonib/httpbackend.py                               |   74 +
 oonib/oonibackend.py                               |   34 +
 oonicli.py                                         |  142 -
 ooniprobe.py                                       |  191 -
 plugins/__init__.py                                |    3 -
 plugins/dropin.cache                               |   48 -
 plugins/skel.py                                    |   20 -
 plugoo/__init__.py                                 |   80 -
 plugoo/assets.py                                   |   56 -
 plugoo/interface.py                                |    2 -
 plugoo/nodes.py                                    |  180 -
 plugoo/reports.py                                  |  175 -
 plugoo/tests.py                                    |  213 -
 plugoo/work.py                                     |  162 -
 skel.py                                            |   10 -
 tests/bridget.py                                   |  378 -
 tests/dnstamper.py                                 |  156 -
 tests/httphost.py                                  |  135 -
 tests/squid.py                                     |  113 -
 tests/tcpscan.py                                   |   86 -
 tests/template.py                                  |   65 -
 tests/traceroute.py                                |  110 -
 unittest/tests.py                                  |   27 -
 utils.py                                           |  146 -
 233 files changed, 41804 insertions(+), 41969 deletions(-)

diff --cc ooni/ooni-probe.conf
index 0000000,336af99..d95e410
mode 000000,100644..100644
--- a/ooni/ooni-probe.conf
+++ b/ooni/ooni-probe.conf
@@@ -1,0 -1,72 +1,77 @@@
+ # ooni-probe
+ #
+ # These are the global configuration parameters necessary to
+ # make ooni-probe work
+ [main]
+ reportdir = reports/
+ logfile = ooniprobe.log
+ assetdir = assets/
+ testdir = oonitests/
+ 
+ loglevel = DEBUG
+ consoleloglevel = DEBUG
+ proxyaddress = 127.0.0.1:9050
+ 
+ # The following configurations are for searching for PlanetLab
+ # nodes, adding them to a slice, and PlanetLab general API
+ # authentication:
+ pl_username = yourusername
+ pl_password = yourpassword
+ 
+ # These are configurations specific to the tests that should be
+ # run by ooni-probe
+ [tests]
+ run = dnstamper
+ ### DNS testing related config parameters
+ 
+ # This is the list of hostnames that must be looked up
+ dns_experiment = top-1m.txt
+ 
+ # This is the dns servers to be tested
+ dns_experiment_dns = dns_servers.txt
+ 
+ # This is the control known good DNS server
 -dns_control_server = 8.8.8.8
++dns_control_server = 91.191.136.152
++
++# Specify whether the dnstamper test should attempt to remove 
++# GeoIP-based false positives by doing a reverse DNS resolve
++# on positive results.
++dns_reverse_lookup = true
+ 
+ ### traceroute testing related config parameters
+ 
+ # This is the list of ips to traceroute to
+ traceroute = example_exp_list.txt
+ 
+ # This is the list of ports that should be used
+ # src_x,src_y,src_z|dst_x,dst_y,dst_z
+ traceroute_ports = 0,53,80,123,443|0,53,80,123,443
+ 
+ # The protocol to be used in the scan
+ traceroute_proto = UDP, TCP, ICMP
+ 
+ ### keyword injection related tests
+ 
+ # List of keywords
+ keywords = keywordlist.txt
+ 
+ # hosts
+ keywords_hosts = hostslist.txt
+ 
+ # Methods to be used for testing
+ keyword_method = http,telnet
+ 
+ ### Tor bridge testing
+ 
+ tor_bridges = bridgetests.txt
+ tor_bridges_timeout = 40
+ 
+ [report]
+ file = report.log
+ timestamp = true
+ #ssh = 127.0.0.1:22
+ #ssh_user = theusername
+ #ssh_password = thepassword
+ #ssh_keyfile = ~/.ssh/mykey_rsa
+ #ssh_rpath = ~/ooni-probe/
+ #tcp = "127.0.0.1:9088"
diff --cc ooni/oonitests/dnstamper.py
index 0000000,68be12d..498ba04
mode 000000,100644..100644
--- a/ooni/oonitests/dnstamper.py
+++ b/ooni/oonitests/dnstamper.py
@@@ -1,0 -1,70 +1,156 @@@
++# -*- coding: utf-8 -*-
++"""
++    dnstamper
++    *********
++
++    This test resolves DNS for a list of domain names, one per line, in the
++    file specified in the ooni-config under the setting "dns_experiment". If
++    the file is top-1m.txt, the test will be run using Amazon's list of top
++    one million domains. The experimental dns servers to query should
++    be specified one per line in assets/dns_servers.txt.
++
++    The test reports censorship if the cardinality of the intersection of
++    the query result set from the control server and the query result set
++    from the experimental server is zero, which is to say, if the two sets
++    have no matching results whatsoever.
++
++    NOTE: This test frequently results in false positives due to GeoIP-based
++    load balancing on major global sites such as google, facebook, and
++    youtube, etc.
++
++    :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft
++    :license: see LICENSE for more details
++"""
++
++try:
++    from dns import resolver, reversename
++except:
++    print "Error: dnspython is not installed! (http://www.dnspython.org/)"
+ try:
 -    from dns import resolver
++    import gevent
+ except:
 -    print "Error dnspython is not installed! (http://www.dnspython.org/)"
 -import gevent
++    print "Error: gevent is not installed! (http://www.gevent.org/)"
++
+ import os
++
+ import plugoo
+ from plugoo.assets import Asset
+ from plugoo.tests import Test
+ 
 -
+ __plugoo__ = "DNST"
+ __desc__ = "DNS censorship detection test"
+ 
++class Top1MAsset(Asset):
++    """
++    Class for parsing top-1m.txt as an asset.
++    """
++    def __init__(self, file=None):
++        self = Asset.__init__(self, file)
++
++    def parse_line(self, line):
++        self = Asset.parse_line(self, line)
++        return line.split(',')[1].replace('\n','')
++
+ class DNSTAsset(Asset):
++    """
++    Creates DNS testing specific Assets.
++    """
+     def __init__(self, file=None):
 -        self = asset.__init__(self, file)
++        self = Asset.__init__(self, file)
+ 
+ class DNST(Test):
+     def lookup(self, hostname, ns):
++        """
++        Resolves a hostname through a DNS nameserver, ns, to the corresponding
++        IP address(es).
++        """
+         res = resolver.Resolver(configure=False)
+         res.nameservers = [ns]
+         answer = res.query(hostname)
+ 
+         ret = []
+ 
+         for data in answer:
+             ret.append(data.address)
+ 
+         return ret
+ 
++    def reverse_lookup(self, ip, ns):
++        """
++        Attempt to do a reverse DNS lookup to determine if the control and exp
++        sets from a positive result resolve to the same domain, in order to
++        remove false positives due to GeoIP load balancing.
++        """
++        res = resolver.Resolver(configure=False)
++        res.nameservers = [ns]
++        n = reversename.from_address(ip)
++        revn = res.query(n, "PTR").__iter__().next().to_text()[:-1]
++
++        return revn
++
+     def experiment(self, *a, **kw):
++        """
++        Compares the lookup() sets of the control and experiment groups.
++        """
+         # this is just a dirty hack
+         address = kw['data'][0]
+         ns = kw['data'][1]
+ 
+         config = self.config
++        ctrl_ns = config.tests.dns_control_server
+ 
+         print "ADDRESS: %s" % address
+         print "NAMESERVER: %s" % ns
+ 
+         exp = self.lookup(address, ns)
 -        control = self.lookup(address, config.tests.dns_control_server)
++        control = self.lookup(address, ctrl_ns)
++
++        result = []
+ 
+         if len(set(exp) & set(control)) > 0:
 -            print "%s : no tampering on %s" % (address, ns)
 -            return (address, ns, False)
++            print "Address %s has not tampered with on DNS server %s\n" % (address, ns)
++            result = (address, ns, exp, control, False)
++            return result
+         else:
 -            print "%s : possible tampering on %s (%s, %s)" % (address, ns, exp, control)
 -            return (address, ns, exp, control, True)
++            print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control)
++            result = (address, ns, exp, control, True)
++
++            if config.tests.dns_reverse_lookup:
++
++                exprevn = [self.reverse_lookup(ip, ns) for ip in exp]
++                ctrlrevn = [self.reverse_lookup(ip, ctrl_ns)
++                            for ip in control]
++
++                if len(set(exprevn) & set(ctrlrevn)) > 0:
++                    print "Further testing has eliminated this as a false positive."
++                else:
++                    print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn)
++                return result
++
++            else:
++                print "\n"
++                return result
+ 
+ def run(ooni):
 -    """Run the test
++    """
++    Run the test.
+     """
+     config = ooni.config
+     urls = []
+ 
 -    dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, \
 -                                            config.tests.dns_experiment))
 -    dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, \
++    if (config.tests.dns_experiment == "top-1m.txt"):
++        dns_experiment = Top1MAsset(os.path.join(config.main.assetdir,
++                                                 config.tests.dns_experiment))
++    else:
++        dns_experiment = DNSTAsset(os.path.join(config.main.assetdir,
++                                                config.tests.dns_experiment))
++    dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir,
+                                                 config.tests.dns_experiment_dns))
+ 
+     assets = [dns_experiment, dns_experiment_dns]
+ 
+     dnstest = DNST(ooni)
 -    ooni.logger.info("starting test")
 -    dnstest.run(assets)
 -    ooni.logger.info("finished")
 -
++    ooni.logger.info("Beginning dnstamper test...")
++    dnstest.run(assets, {'index': 1})
++    ooni.logger.info("Dnstamper test completed!")
+ 
diff --cc ooni/oonitests/httphost.py
index 0000000,6446e1f..25adff2
mode 000000,100644..100644
--- a/ooni/oonitests/httphost.py
+++ b/ooni/oonitests/httphost.py
@@@ -1,0 -1,132 +1,135 @@@
+ """
+     HTTP Host based filtering
+     *************************
+ 
+     This test detect HTTP Host field
+     based filtering.
+     It is used to detect censorship on
+     performed with Web Guard (used by
+     T-Mobile US).
+ """
+ import os
+ from datetime import datetime
+ from gevent import monkey
+ 
+ import urllib2
+ import httplib
+ # WARNING! Using gevent's socket
+ # introduces the 0x20 DNS "feature".
+ # This will result is weird DNS requests
+ # appearing on the wire.
+ monkey.patch_socket()
+ 
 -from BeautifulSoup import BeautifulSoup
++try:
++    from BeautifulSoup import BeautifulSoup
++except:
++    print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/";
+ 
+ from plugoo.assets import Asset
+ from plugoo.tests import Test
+ 
+ __plugoo__ = "HTTP Host"
+ __desc__ = "This detects HTTP Host field based filtering"
+ 
+ class HTTPHostAsset(Asset):
+     """
+     This is the asset that should be used by the Test. It will
+     contain all the code responsible for parsing the asset file
+     and should be passed on instantiation to the test.
+     """
+     def __init__(self, file=None):
+         self = Asset.__init__(self, file)
+ 
+     def parse_line(self, line):
+         return line.split(',')[1].replace('\n','')
+ 
+ class HTTPHost(Test):
+     """
+     The main Test class
+     """
+ 
+     def check_response(self, response):
+         soup = BeautifulSoup(response)
+         if soup.head.title.string == "Content Filtered":
+             # Response indicates censorship
+             return True
+         else:
+             # Response does not indicate censorship
+             return False
+ 
+ 
+     def is_censored(self, response):
+         if response:
+             soup = BeautifulSoup(response)
+             censored = self.check_response(response)
+         else:
+             censored = "unreachable"
+         return censored
+ 
+     def urllib2_test(self, control_server, host):
+         req = urllib2.Request(control_server)
+         req.add_header('Host', host)
+         try:
+             r = urllib2.urlopen(req)
+             response = r.read()
+             censored = self.is_censored(response)
+         except Exception, e:
+             censored = "Error! %s" % e
+ 
+         return censored
+ 
+     def httplib_test(self, control_server, host):
+         try:
+             conn = httplib.HTTPConnection(control_server)
+             conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True)
+             conn.putheader("Host", host)
+             conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6")
+             conn.endheaders()
+             r = conn.getresponse()
+             response = r.read()
+             censored = self.is_censored(response)
+         except Exception, e:
+             censored = "Error! %s" % e
+ 
+         return censored
+ 
+ 
+     def experiment(self, *a, **kw):
+         """
+         Try to connect to the control server with
+         the specified host field.
+         """
+         host = kw['data']
+         control_server = kw['control_server']
+         self.logger.info("Testing %s (%s)" % (host, control_server))
+ 
+         #censored = self.urllib2_test(control_server, host)
+         censored = self.httplib_test(control_server, host)
+ 
+         self.logger.info("%s: %s" % (host, censored))
+         return {'Time': datetime.now(),
+                 'Host': host,
+                 'Censored': censored}
+ 
+ 
+ def run(ooni):
+     """
+     This is the function that will be called by OONI
+     and it is responsible for instantiating and passing
+     the arguments to the Test class.
+     """
+     config = ooni.config
+ 
+     # This the assets array to be passed to the run function of
+     # the test
+     assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \
+                                             "top-1m.csv"))]
+ 
+     # Instantiate the Test
+     thetest = HTTPHost(ooni)
+     ooni.logger.info("starting HTTP Host Test...")
+     # Run the test with argument assets
+     thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'})
+     ooni.logger.info("finished.")
+ 
+ 
diff --cc ooni/plugoo/reports.py
index 0000000,c099456..c3e7c23
mode 000000,100644..100644
--- a/ooni/plugoo/reports.py
+++ b/ooni/plugoo/reports.py
@@@ -1,0 -1,175 +1,175 @@@
+ import os
+ from datetime import datetime
+ import yaml
+ 
+ import logging
+ import itertools
+ import gevent
+ 
+ class Report:
+     """This is the ooni-probe reporting mechanism. It allows
+     reporting to multiple destinations and file formats.
+ 
+     :scp the string of <host>:<port> of an ssh server
+ 
+     :yaml the filename of a the yaml file to write
+ 
+     :file the filename of a simple txt file to write
+ 
+     :tcp the <host>:<port> of a TCP server that will just listen for
+          inbound connection and accept a stream of data (think of it
+          as a `nc -l -p <port> > filename.txt`)
+     """
+     def __init__(self, ooni,
+                  scp="127.0.0.1:22",
+                  file="test.report",
+                  tcp="127.0.0.1:9000"):
+ 
+         self.file = file
+         self.tcp = tcp
+         self.scp = scp
+         self.config = ooni.config.report
+         self.logger = ooni.logger
+ 
+         if self.config.timestamp:
+             tmp = self.file.split('.')
+             self.file = '.'.join(tmp[:-1]) + "-" + \
+                         datetime.now().isoformat('-') + '.' + \
+                         tmp[-1]
+             print self.file
+ 
+         try:
+             import paramiko
+         except:
+             self.scp = None
 -            self.logger.warn("Could not import paramiko. SCP will not be disabled")
++            self.logger.warn("Could not import paramiko. SCP will be disabled")
+ 
+     def __call__(self, data):
+         """
+         This should be invoked every time you wish to write some
+         data to the reporting system
+         """
+         #print "Writing report(s)"
+         #dump = '--- \n'
+         dump = yaml.dump([data])
+         #dump += yaml.dump(data)
+         reports = []
+ 
+         if self.file:
+             reports.append("file")
+ 
+         if self.tcp:
+             reports.append("tcp")
+ 
+         if self.scp:
+             reports.append("scp")
+ 
+         jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports]
+         gevent.joinall(jobs)
+         ret = []
+         for job in jobs:
+             #print job.value
+             ret.append(job.value)
+         return ret
+ 
+     def file_report(self, data, file=None, mode='a+'):
+         """
+         This reports to a file in YAML format
+         """
+         if not file:
+             file = self.file
+         try:
+             f = open(file, mode)
+             f.write(data)
+         except Exception, e:
+             raise e
+         finally:
+             f.close()
+ 
+ 
+     def tcp_report(self, data):
+         """
+         This connect to the specified tcp server
+         and writes the data passed as argument.
+         """
+         host, port = self.tcp.split(":")
+         tcp = socket.getprotobyname('tcp')
+         send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, tcp)
+         try:
+             send_socket.connect((host, int(port)))
+             send_socket.send(data)
+ 
+         except Exception, e:
+             raise e
+ 
+         finally:
+             send_socket.close()
+ 
+ 
+     def scp_report(self, data, rfile=None, mode='a+'):
+         """
+         Push data to the remote ssh server.
+ 
+         :rfile the remote filename to write
+         :data the raw data content that should be written
+         :mode in what mode the file should be created
+         """
+         if not rfile:
+             rfile = self.file
+         host, port = self.scp.split(":")
+         transport = paramiko.Transport((host, port))
+ 
+         # The remote path of the remote file to write
+         rfpath = os.path.join(self.config.ssh_rpath, rfile)
+ 
+         try:
+             username = self.config.ssh_username
+         except:
+             raise "No username provided"
+ 
+         # Load the local known host key file
+         transport.load_host_keys(os.path.expanduser("~/.ssh/known_hosts"))
+ 
+         # We prefer to use an ssh keyfile fo authentication
+         if self.config.ssh_keyfile:
+             keyfile = os.path.expanduser(self.config.ssh_keyfile)
+             key = paramiko.RSAKey.from_private_key_file(keylocfile)
+             try:
+                 transport.connect(username=username, pkey=key)
+             except Exception, e:
+                 raise e
+ 
+         # If not even a password is fine
+         elif self.config.ssh_password:
+             try:
+                 transport.connect(username=username, password=self.config.ssh_password)
+             except Exception, e:
+                 raise e
+ 
+         # ... but no authentication, that is madness!
+         else:
+             raise "No key or password provided for ssh"
+ 
+         sftp = paramiko.SFTPClient.from_transport(transport)
+         try:
+             sftp = ssh.open_sftp()
+             remote_file = sftp.file(rfile, mode)
+             remote_file.set_pipelined(True)
+             remote_file.write(data)
+ 
+         except Exception, e:
+             raise e
+         sftp.close()
+         transport.close()
+ 
+ 
+     def send_report(self, data, type):
+         """
+         This sends the report using the
+         specified type.
+         """
+         #print "Reporting %s to %s" % (data, type)
+         self.logger.info("Reporting to %s" % type)
+         getattr(self, type+"_report").__call__(data)
+ 
+ 



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits