[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Refactor reporter and removal of dead code



commit 6bb22e9b86a073cdbd9dc87d7ea87389f7296239
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Wed Oct 24 15:59:53 2012 +0000

    Refactor reporter and removal of dead code
---
 ooni/oonicli.py            |   27 ++++-----
 ooni/oonitests/httphost.py |  135 --------------------------------------------
 ooni/oonitests/squid.py    |  113 ------------------------------------
 ooni/oonitests/template.py |   65 ---------------------
 ooni/reporter.py           |   26 ++++++---
 5 files changed, 29 insertions(+), 337 deletions(-)

diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 5ea92d7..aa3cdc0 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -13,29 +13,24 @@
 #    original copyright (c) by Twisted Matrix Laboratories.
 
 
-import sys, os, random, gc, time, warnings
-
-import unittest
-import inspect
-
-from ooni.inputunit import InputUnitFactory
-from ooni.reporter import ReporterFactory
-from ooni.nettest import InputTestSuite
-from ooni.plugoo import tests
-from ooni import nettest, runner, reporter
+import sys
+import os
+import random
+import time
 
 from twisted.internet import defer
 from twisted.application import app
-from twisted.python import usage, reflect, failure
-from twisted.python.filepath import FilePath
-from twisted import plugin
+from twisted.python import usage, failure
 from twisted.python.util import spewer
-from twisted.python.compat import set
-from twisted.trial import itrial
-from twisted.trial import runner as irunner
 
+from ooni import nettest, runner, reporter
+
+from ooni.inputunit import InputUnitFactory
+from ooni.reporter import ReporterFactory
+from ooni.nettest import InputTestSuite
 from ooni.utils import log
 
+
 class Options(usage.Options, app.ReactorSelectionMixin):
     synopsis = """%s [options] [[file|package|module|TestCase|testmethod]...]
     """ % (os.path.basename(sys.argv[0]),)
diff --git a/ooni/oonitests/__init__.py b/ooni/oonitests/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/ooni/oonitests/httphost.py b/ooni/oonitests/httphost.py
deleted file mode 100644
index 25adff2..0000000
--- a/ooni/oonitests/httphost.py
+++ /dev/null
@@ -1,135 +0,0 @@
-"""
-    HTTP Host based filtering
-    *************************
-
-    This test detect HTTP Host field
-    based filtering.
-    It is used to detect censorship on
-    performed with Web Guard (used by
-    T-Mobile US).
-"""
-import os
-from datetime import datetime
-from gevent import monkey
-
-import urllib2
-import httplib
-# WARNING! Using gevent's socket
-# introduces the 0x20 DNS "feature".
-# This will result is weird DNS requests
-# appearing on the wire.
-monkey.patch_socket()
-
-try:
-    from BeautifulSoup import BeautifulSoup
-except:
-    print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/";
-
-from plugoo.assets import Asset
-from plugoo.tests import Test
-
-__plugoo__ = "HTTP Host"
-__desc__ = "This detects HTTP Host field based filtering"
-
-class HTTPHostAsset(Asset):
-    """
-    This is the asset that should be used by the Test. It will
-    contain all the code responsible for parsing the asset file
-    and should be passed on instantiation to the test.
-    """
-    def __init__(self, file=None):
-        self = Asset.__init__(self, file)
-
-    def parse_line(self, line):
-        return line.split(',')[1].replace('\n','')
-
-class HTTPHost(Test):
-    """
-    The main Test class
-    """
-
-    def check_response(self, response):
-        soup = BeautifulSoup(response)
-        if soup.head.title.string == "Content Filtered":
-            # Response indicates censorship
-            return True
-        else:
-            # Response does not indicate censorship
-            return False
-
-
-    def is_censored(self, response):
-        if response:
-            soup = BeautifulSoup(response)
-            censored = self.check_response(response)
-        else:
-            censored = "unreachable"
-        return censored
-
-    def urllib2_test(self, control_server, host):
-        req = urllib2.Request(control_server)
-        req.add_header('Host', host)
-        try:
-            r = urllib2.urlopen(req)
-            response = r.read()
-            censored = self.is_censored(response)
-        except Exception, e:
-            censored = "Error! %s" % e
-
-        return censored
-
-    def httplib_test(self, control_server, host):
-        try:
-            conn = httplib.HTTPConnection(control_server)
-            conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True)
-            conn.putheader("Host", host)
-            conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6")
-            conn.endheaders()
-            r = conn.getresponse()
-            response = r.read()
-            censored = self.is_censored(response)
-        except Exception, e:
-            censored = "Error! %s" % e
-
-        return censored
-
-
-    def experiment(self, *a, **kw):
-        """
-        Try to connect to the control server with
-        the specified host field.
-        """
-        host = kw['data']
-        control_server = kw['control_server']
-        self.logger.info("Testing %s (%s)" % (host, control_server))
-
-        #censored = self.urllib2_test(control_server, host)
-        censored = self.httplib_test(control_server, host)
-
-        self.logger.info("%s: %s" % (host, censored))
-        return {'Time': datetime.now(),
-                'Host': host,
-                'Censored': censored}
-
-
-def run(ooni):
-    """
-    This is the function that will be called by OONI
-    and it is responsible for instantiating and passing
-    the arguments to the Test class.
-    """
-    config = ooni.config
-
-    # This the assets array to be passed to the run function of
-    # the test
-    assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \
-                                            "top-1m.csv"))]
-
-    # Instantiate the Test
-    thetest = HTTPHost(ooni)
-    ooni.logger.info("starting HTTP Host Test...")
-    # Run the test with argument assets
-    thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'})
-    ooni.logger.info("finished.")
-
-
diff --git a/ooni/oonitests/squid.py b/ooni/oonitests/squid.py
deleted file mode 100644
index 847135e..0000000
--- a/ooni/oonitests/squid.py
+++ /dev/null
@@ -1,113 +0,0 @@
-"""
-    Squid Proxy Detector
-    ********************
-
-"""
-import os
-import httplib
-import urllib2
-from urlparse import urlparse
-
-from plugoo import gen_headers
-from plugoo.assets import Asset
-from plugoo.tests import Test
-
-__plugoo__ = "SquidProxy"
-__desc__ = "This Test aims at detecting the squid transparent proxy"
-
-class SquidAsset(Asset):
-    """
-    This is the asset that should be used by the Test. It will
-    contain all the code responsible for parsing the asset file
-    and should be passed on instantiation to the test.
-    """
-    def __init__(self, file=None):
-        self = Asset.__init__(self, file)
-
-
-class Squid(Test):
-    """
-    Squid Proxy testing class.
-    """
-    def _http_request(self, method, url,
-                      path=None, headers=None):
-        """
-        Perform an HTTP Request.
-        XXX move this function to the core OONI
-        code.
-        """
-        url = urlparse(url)
-        host = url.netloc
-
-        conn = httplib.HTTPConnection(host, 80)
-        conn.connect()
-
-        if path is None:
-            path = purl.path
-
-        conn.putrequest(method, path)
-
-        for h in gen_headers():
-            conn.putheaders(h[0], h[1])
-        conn.endheaders()
-
-        send_browser_headers(self, None, conn)
-
-        response = conn.getresponse()
-
-        headers = dict(response.getheaders())
-
-        self.headers = headers
-        self.data = response.read()
-        return True
-
-    def invalid_request(self):
-        """
-        This will trigger squids "Invalid Request" error.
-        """
-        pass
-
-    def cache_object(self):
-        """
-        This attempts to do a GET cache_object://localhost/info on
-        any destination and checks to see if the response contains
-        is that of Squid.
-        """
-
-        pass
-
-    def experiment(self, *a, **kw):
-        """
-        Fill this up with the tasks that should be performed
-        on the "dirty" network and should be compared with the
-        control.
-        """
-
-
-    def control(self):
-        """
-        Fill this up with the control related code.
-        """
-        return True
-
-def run(ooni):
-    """
-    This is the function that will be called by OONI
-    and it is responsible for instantiating and passing
-    the arguments to the Test class.
-    """
-    config = ooni.config
-
-    # This the assets array to be passed to the run function of
-    # the test
-    assets = [TestTemplateAsset(os.path.join(config.main.assetdir, \
-                                            "someasset.txt"))]
-
-    # Instantiate the Test
-    thetest = TestTemplate(ooni)
-    ooni.logger.info("starting SquidProxyTest...")
-    # Run the test with argument assets
-    thetest.run(assets)
-    ooni.logger.info("finished.")
-
-
diff --git a/ooni/oonitests/template.py b/ooni/oonitests/template.py
deleted file mode 100644
index f5f9d93..0000000
--- a/ooni/oonitests/template.py
+++ /dev/null
@@ -1,65 +0,0 @@
-"""
-    Test Template
-    *************
-
-    This is a Test template, to be used when writing your
-    own OONI probe Tests.
-"""
-import os
-
-from plugoo.assets import Asset
-from plugoo.tests import Test
-
-__plugoo__ = "Test Template"
-__desc__ = "This a test template to be used to build your own tests"
-
-class TestTemplateAsset(Asset):
-    """
-    This is the asset that should be used by the Test. It will
-    contain all the code responsible for parsing the asset file
-    and should be passed on instantiation to the test.
-    """
-    def __init__(self, file=None):
-        self = Asset.__init__(self, file)
-
-
-class TestTemplate(Test):
-    """
-    The main Test class
-    """
-
-    def experiment(self, *a, **kw):
-        """
-        Fill this up with the tasks that should be performed
-        on the "dirty" network and should be compared with the
-        control.
-        """
-        return False
-
-    def control(self):
-        """
-        Fill this up with the control related code.
-        """
-        return True
-
-def run(ooni):
-    """
-    This is the function that will be called by OONI
-    and it is responsible for instantiating and passing
-    the arguments to the Test class.
-    """
-    config = ooni.config
-
-    # This the assets array to be passed to the run function of
-    # the test
-    assets = [TestTemplateAsset(os.path.join(config.main.assetdir, \
-                                            "someasset.txt"))]
-
-    # Instantiate the Test
-    thetest = TestTemplate(ooni)
-    ooni.logger.info("starting TestTemplate...")
-    # Run the test with argument assets
-    thetest.run(assets)
-    ooni.logger.info("finished.")
-
-
diff --git a/ooni/reporter.py b/ooni/reporter.py
index c12b28f..fc5dde9 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -1,4 +1,3 @@
-import copy_reg
 import itertools
 import logging
 import sys
@@ -12,8 +11,8 @@ from yaml.serializer import *
 from yaml.resolver import *
 
 from datetime import datetime
-from twisted.python.util import OrderedDict, untilConcludes
-from twisted.trial import unittest, reporter, runner
+from twisted.python.util import untilConcludes
+from twisted.trial import reporter
 from twisted.internet import defer
 from ooni.utils import date, log, geodata
 
@@ -30,6 +29,11 @@ except:
 pyunit =  __import__('unittest')
 
 class OSafeRepresenter(SafeRepresenter):
+    """
+    This is a custom YAML representer that allows us to represent reports
+    safely.
+    It extends the SafeRepresenter to be able to also represent complex numbers
+    """
     def represent_complex(self, data):
         if data.imag == 0.0:
             data = u'%r' % data.real
@@ -45,7 +49,10 @@ OSafeRepresenter.add_representer(complex,
                                  OSafeRepresenter.represent_complex)
 
 class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver):
-
+    """
+    This is a modification of the YAML Safe Dumper to use our own Safe
+    Representer that supports complex numbers.
+    """
     def __init__(self, stream,
             default_style=None, default_flow_style=None,
             canonical=None, indent=None, width=None,
@@ -64,6 +71,9 @@ class OSafeDumper(Emitter, Serializer, OSafeRepresenter, Resolver):
 
 
 def safe_dump(data, stream=None, **kw):
+    """
+    Safely dump to a yaml file the specified data.
+    """
     return yaml.dump_all([data], stream, Dumper=OSafeDumper, **kw)
 
 class OReporter(pyunit.TestResult):
@@ -88,8 +98,8 @@ class OReporter(pyunit.TestResult):
     def _getTime(self):
         return time.time()
 
-    def _write(self, format, *args):
-        s = str(format)
+    def _write(self, format_string, *args):
+        s = str(format_string)
         assert isinstance(s, type(''))
         if args:
             self._stream.write(s % args)
@@ -97,8 +107,8 @@ class OReporter(pyunit.TestResult):
             self._stream.write(s)
         untilConcludes(self._stream.flush)
 
-    def _writeln(self, format, *args):
-        self._write(format, *args)
+    def _writeln(self, format_string, *args):
+        self._write(format_string, *args)
         self._write('\n')
 
     def writeYamlLine(self, line):



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits