[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Do some major refactoring of the ooniprobe backend



commit dce255af61ae3ee89a0d37776103919dd86e4374
Author: Arturo Filastò <art@xxxxxxxxx>
Date:   Sun Nov 11 15:41:28 2012 +0100

    Do some major refactoring of the ooniprobe backend
    * Implement TCP Echo test helper
    * Refactor testhelpers into proper namespace
    * Remove unused imports
    * Remove files that were not being imported anywhere
---
 oonib/config.py                   |   39 ++++++++++++------
 oonib/lib/__init__.py             |    1 -
 oonib/lib/ssl.py                  |    8 ----
 oonib/oonibackend.conf.sample     |   12 ------
 oonib/oonibackend.py              |   79 ++++++++++++++++++++----------------
 oonib/testhelpers/__init__.py     |    5 ++
 oonib/testhelpers/dns.py          |   16 -------
 oonib/testhelpers/dns_helpers.py  |   16 +++++++
 oonib/testhelpers/http_helpers.py |   80 +++++++++++++++++++++++++++++++++++++
 oonib/testhelpers/httph.py        |   80 -------------------------------------
 oonib/testhelpers/ssl_helpers.py  |    9 ++++
 oonib/testhelpers/tcp_helpers.py  |   14 ++++++
 ooniprobe.conf                    |    3 +-
 13 files changed, 195 insertions(+), 167 deletions(-)

diff --git a/oonib/config.py b/oonib/config.py
index 0156b69..dc2be2f 100644
--- a/oonib/config.py
+++ b/oonib/config.py
@@ -9,21 +9,34 @@ def get_root_path():
 
 # XXX convert this to something that is a proper config file
 main = Storage()
-main.reporting_port = 8888
-main.http_port = 8080
-main.dns_udp_port = 5354
-main.dns_tcp_port = 8002
-main.daphn3_port = 9666
-main.server_version = "Apache"
+main.collector_port = 8888
+
+# XXX make this be the server name that is of 
 main.database_uri = "sqlite:"+get_root_path()+"oonib_test_db.db"
 main.db_threadpool_size = 10
-#main.ssl_private_key = /path/to/data/private.key
-#main.ssl_certificate = /path/to/data/certificate.crt
-#main.ssl_port = 8433
 
 helpers = Storage()
-helpers.http_return_request_port = 1234
 
-daphn3 = Storage()
-daphn3.yaml_file = "/path/to/data/oonib/daphn3.yaml"
-daphn3.pcap_file = "/path/to/data/server.pcap"
+helpers.http_return_request = Storage()
+helpers.http_return_request.port = 57001 
+helpers.http_return_request.server_version = "Apache"
+
+helpers.tcp_echo = Storage()
+helpers.tcp_echo.port = 57002 
+
+helpers.daphn3 = Storage()
+helpers.daphn3.yaml_file = "/path/to/data/oonib/daphn3.yaml"
+helpers.daphn3.pcap_file = "/path/to/data/server.pcap"
+
+helpers.daphn3.port = 57003
+
+helpers.dns = Storage()
+helpers.dns.udp_port = 57004
+helpers.dns.tcp_port = 57005
+
+helpers.ssl = Storage()
+#helpers.ssl.private_key = /path/to/data/private.key
+#helpers.ssl.certificate = /path/to/data/certificate.crt
+#helpers.ssl.port = 57007
+
+
diff --git a/oonib/lib/__init__.py b/oonib/lib/__init__.py
deleted file mode 100644
index 8b13789..0000000
--- a/oonib/lib/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/oonib/lib/ssl.py b/oonib/lib/ssl.py
deleted file mode 100644
index 20e99e8..0000000
--- a/oonib/lib/ssl.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from twisted.internet import ssl
-from oonib import config
-
-class SSLContext(ssl.DefaultOpenSSLContextFactory):
-    def __init__(self, *args, **kw):
-        ssl.DefaultOpenSSLContextFactory.__init__(self, config.main.ssl_private_key,
-                                                  config.main.ssl_certificate)
-
diff --git a/oonib/oonibackend.conf.sample b/oonib/oonibackend.conf.sample
deleted file mode 100644
index dc0a662..0000000
--- a/oonib/oonibackend.conf.sample
+++ /dev/null
@@ -1,12 +0,0 @@
-[main]
-reporting_port = 8888
-http_port = 8080
-dns_udp_port = 5354
-dns_tcp_port = 8002
-daphn3_port = 9666
-server_version = Apache
-ssl_private_key = /path/to/private.key
-ssl_certificate = /path/to/certificate.crt
-[daphn3]
-pcap_file = /path/to/server.pcap
-yaml_file = /path/to/server.yaml
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 799dcf0..8f47e73 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -3,57 +3,53 @@
 #
 # This is the backend system responsible for running certain services that are
 # useful for censorship detection.
-
-import json
-import random
-import string
+#
+# In here we start all the test helpers that are required by ooniprobe and
+# start the report collector
 
 from twisted.application import internet
-
-from twisted.internet import protocol, reactor, defer
+from twisted.internet import  reactor
 from twisted.application import internet, service
 from twisted.application.service import Application
-
-from twisted.web import resource, server, static
-from twisted.web.microdom import escape
-from twisted.protocols import basic
 from twisted.names import dns
 
 from ooni.utils import log
 
 from oonib.report.api import reportingBackend
-from oonib.lib.ssl import SSLContext
+
 from oonib import config
 
-from oonib.testhelpers.httph import HTTPReturnJSONHeadersHelper
-from oonib.testhelpers.dns import ProxyDNSServer
+from oonib.testhelpers import dns_helpers, ssl_helpers
+from oonib.testhelpers import http_helpers, tcp_helpers
+
 #from oonib.testhelpers.daphn3 import Daphn3Server
 from oonib import db_threadpool
 
 from cyclone import web
 
-# This tells twisted to set the
-server.version = config.main.server_version
-
 application = service.Application('oonibackend')
 serviceCollection = service.IServiceCollection(application)
 
-if config.main.ssl_port:
-    internet.SSLServer(int(config.main.ssl_port),
+if config.helpers.ssl.port:
+    log.msg("Starting SSL helper")
+    ssl_helper = internet.SSLServer(int(config.helpers.ssl.port),
                    server.Site(HTTPBackend()),
-                   SSLContext(config),
-                  ).setServiceParent(serviceCollection)
+                   SSLContext(config))
+    ssl_helper.setServiceParent(serviceCollection)
 
 # Start the DNS Server related services
-if config.main.dns_tcp_port:
-    TCPDNSServer = ProxyDNSServer()
-    internet.TCPServer(int(config.main.dns_tcp_port),
-                       TCPDNSServer).setServiceParent(serviceCollection)
-
-if config.main.dns_udp_port:
-    UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer)
-    internet.UDPServer(int(config.main.dns_udp_port),
-                       UDPFactory).setServiceParent(serviceCollection)
+if config.helpers.dns.tcp_port:
+    log.msg("Starting TCP DNS Helper on %s" % config.helpers.dns.tcp_port)
+    tcp_dns_helper = internet.TCPServer(int(config.helpers.dns.tcp_port),
+                       dns_helpers.DNSTestHelper())
+    tcp_dns_helper.setServiceParent(serviceCollection)
+
+if config.helpers.dns.udp_port:
+    log.msg("Starting UDP DNS Helper on %s" % config.helpers.dns.udp_port)
+    udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper())
+    udp_dns_helper = internet.UDPServer(int(config.helpers.dns.udp_port),
+                       udp_dns_factory)
+    udp_dns_helper.setServiceParent(serviceCollection)
 
 # XXX this needs to be ported
 # Start the OONI daphn3 backend
@@ -62,12 +58,23 @@ if config.main.dns_udp_port:
 #    internet.TCPServer(int(config.main.daphn3_port),
 #                       daphn3).setServiceParent(serviceCollection)
 
-if config.main.reporting_port:
-    internet.TCPServer(int(config.main.reporting_port),
-                       reportingBackend).setServiceParent(serviceCollection)
-
-if config.helpers.http_return_request_port:
-    internet.TCPServer(int(config.helpers.http_return_request_port),
-            HTTPReturnJSONHeadersHelper).setServiceParent(serviceCollection)
+if config.main.collector_port:
+    log.msg("Starting Collector on %s" % config.main.collector_port)
+    collector = internet.TCPServer(int(config.main.collector_port),
+                       reportingBackend)
+    collector.setServiceParent(serviceCollection)
+
+if config.helpers.tcp_echo.port:
+    log.msg("Starting TCP echo helper on %s" % config.helpers.tcp_echo.port)
+    tcp_echo_helper = internet.TCPServer(int(config.helpers.tcp_echo.port),
+                        tcp_helpers.TCPEchoHelper())
+    tcp_echo_helper.setServiceParent(serviceCollection)
+
+if config.helpers.http_return_request.port:
+    log.msg("Starting HTTP return request helper on %s" % config.helpers.http_return_request.port)
+    http_return_request_helper = internet.TCPServer(
+            int(config.helpers.http_return_request.port),
+            http_helpers.HTTPReturnJSONHeadersHelper)
+    http_return_request_helper.setServiceParent(serviceCollection)
 
 reactor.addSystemEventTrigger('after', 'shutdown', db_threadpool.stop)
diff --git a/oonib/testhelpers/__init__.py b/oonib/testhelpers/__init__.py
index e69de29..4dbb547 100644
--- a/oonib/testhelpers/__init__.py
+++ b/oonib/testhelpers/__init__.py
@@ -0,0 +1,5 @@
+from . import dns_helpers
+from . import http_helpers
+from . import tcp_helpers
+
+__all__ = ['dns_helpers', 'http_helpers', 'tcp_helpers']
diff --git a/oonib/testhelpers/dns.py b/oonib/testhelpers/dns.py
deleted file mode 100644
index 689d1a4..0000000
--- a/oonib/testhelpers/dns.py
+++ /dev/null
@@ -1,16 +0,0 @@
-from twisted.internet.protocol import Factory, Protocol
-from twisted.internet import reactor
-from twisted.names import dns
-from twisted.names import client, server
-
-class ProxyDNSServer(server.DNSServerFactory):
-    def __init__(self, authorities = None,
-                 caches = None, clients = None,
-                 verbose = 0):
-        resolver = client.Resolver(servers=[('8.8.8.8', 53)])
-        server.DNSServerFactory.__init__(self, authorities = authorities,
-                                         caches = caches, clients = [resolver],
-                                         verbose = verbose)
-    def handleQuery(self, message, protocol, address):
-        print message, protocol, address
-        server.DNSServerFactory.handleQuery(self, message, protocol, address)
diff --git a/oonib/testhelpers/dns_helpers.py b/oonib/testhelpers/dns_helpers.py
new file mode 100644
index 0000000..cb4ff9f
--- /dev/null
+++ b/oonib/testhelpers/dns_helpers.py
@@ -0,0 +1,16 @@
+from twisted.internet.protocol import Factory, Protocol
+from twisted.internet import reactor
+from twisted.names import dns
+from twisted.names import client, server
+
+class DNSTestHelper(server.DNSServerFactory):
+    def __init__(self, authorities = None,
+                 caches = None, clients = None,
+                 verbose = 0):
+        resolver = client.Resolver(servers=[('8.8.8.8', 53)])
+        server.DNSServerFactory.__init__(self, authorities = authorities,
+                                         caches = caches, clients = [resolver],
+                                         verbose = verbose)
+    def handleQuery(self, message, protocol, address):
+        print message, protocol, address
+        server.DNSServerFactory.handleQuery(self, message, protocol, address)
diff --git a/oonib/testhelpers/http_helpers.py b/oonib/testhelpers/http_helpers.py
new file mode 100644
index 0000000..7001fa6
--- /dev/null
+++ b/oonib/testhelpers/http_helpers.py
@@ -0,0 +1,80 @@
+import json
+import random
+import string
+
+from twisted.application import internet, service
+from twisted.internet import protocol, reactor, defer
+from twisted.protocols import basic
+from twisted.web import resource, server, static, http
+from twisted.web.microdom import escape
+
+from cyclone.web import RequestHandler, Application
+
+class HTTPTrapAll(RequestHandler):
+    """
+    Master class to be used to trap all the HTTP methods and make capitalized
+    requests pass.
+    """
+    def _execute(self, transforms, *args, **kwargs):
+        self._transforms = transforms
+        defer.maybeDeferred(self.prepare).addCallbacks(
+                    self._execute_handler,
+                    lambda f: self._handle_request_exception(f.value),
+                    callbackArgs=(args, kwargs))
+
+    def _execute_handler(self, r, args, kwargs):
+        if not self._finished:
+            args = [self.decode_argument(arg) for arg in args]
+            kwargs = dict((k, self.decode_argument(v, name=k))
+                            for (k, v) in kwargs.iteritems())
+            # This is where we do the patching
+            # XXX this is somewhat hackish
+            d = defer.maybeDeferred(self.all, *args, **kwargs)
+            d.addCallbacks(self._execute_success, self._execute_failure)
+            self.notifyFinish().addCallback(self.on_connection_close)
+
+class HTTPReturnJSONHeaders(HTTPTrapAll):
+    def all(self):
+        # XXX make sure that the request headers are in the correct order
+        submitted_data = {'request_body': self.request.body,
+                'request_headers': self.request.headers,
+                'request_uri': self.request.uri,
+                'request_method': self.request.method}
+        response = json.dumps(submitted_data)
+        self.write(response)
+
+HTTPReturnJSONHeadersHelper = Application([
+    (r"/*", HTTPReturnJSONHeaders)
+])
+
+
+class HTTPRandomPage(HTTPTrapAll):
+    """
+    This generates a random page of arbitrary length and containing the string
+    selected by the user.
+    /<length>/<keyword>
+    XXX this is currently disabled as it is not of use to any test.
+    """
+    isLeaf = True
+    def _gen_random_string(self, length):
+        return ''.join(random.choice(string.letters) for x in range(length))
+
+    def genRandomPage(self, length=100, keyword=None):
+        data = self._gen_random_string(length/2)
+        if keyword:
+            data += keyword
+        data += self._gen_random_string(length - length/2)
+        data += '\n'
+        return data
+
+    def all(self, length, keyword):
+        length = 100
+        if length > 100000:
+            length = 100000
+        return self.genRandomPage(length, keyword)
+
+HTTPRandomPageHelper = Application([
+    # XXX add regexps here
+    (r"/(.*)/(.*)", HTTPRandomPage)
+])
+
diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py
deleted file mode 100644
index 7001fa6..0000000
--- a/oonib/testhelpers/httph.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import json
-import random
-import string
-
-from twisted.application import internet, service
-from twisted.internet import protocol, reactor, defer
-from twisted.protocols import basic
-from twisted.web import resource, server, static, http
-from twisted.web.microdom import escape
-
-from cyclone.web import RequestHandler, Application
-
-class HTTPTrapAll(RequestHandler):
-    """
-    Master class to be used to trap all the HTTP methods and make capitalized
-    requests pass.
-    """
-    def _execute(self, transforms, *args, **kwargs):
-        self._transforms = transforms
-        defer.maybeDeferred(self.prepare).addCallbacks(
-                    self._execute_handler,
-                    lambda f: self._handle_request_exception(f.value),
-                    callbackArgs=(args, kwargs))
-
-    def _execute_handler(self, r, args, kwargs):
-        if not self._finished:
-            args = [self.decode_argument(arg) for arg in args]
-            kwargs = dict((k, self.decode_argument(v, name=k))
-                            for (k, v) in kwargs.iteritems())
-            # This is where we do the patching
-            # XXX this is somewhat hackish
-            d = defer.maybeDeferred(self.all, *args, **kwargs)
-            d.addCallbacks(self._execute_success, self._execute_failure)
-            self.notifyFinish().addCallback(self.on_connection_close)
-
-class HTTPReturnJSONHeaders(HTTPTrapAll):
-    def all(self):
-        # XXX make sure that the request headers are in the correct order
-        submitted_data = {'request_body': self.request.body,
-                'request_headers': self.request.headers,
-                'request_uri': self.request.uri,
-                'request_method': self.request.method}
-        response = json.dumps(submitted_data)
-        self.write(response)
-
-HTTPReturnJSONHeadersHelper = Application([
-    (r"/*", HTTPReturnJSONHeaders)
-])
-
-
-class HTTPRandomPage(HTTPTrapAll):
-    """
-    This generates a random page of arbitrary length and containing the string
-    selected by the user.
-    /<length>/<keyword>
-    XXX this is currently disabled as it is not of use to any test.
-    """
-    isLeaf = True
-    def _gen_random_string(self, length):
-        return ''.join(random.choice(string.letters) for x in range(length))
-
-    def genRandomPage(self, length=100, keyword=None):
-        data = self._gen_random_string(length/2)
-        if keyword:
-            data += keyword
-        data += self._gen_random_string(length - length/2)
-        data += '\n'
-        return data
-
-    def all(self, length, keyword):
-        length = 100
-        if length > 100000:
-            length = 100000
-        return self.genRandomPage(length, keyword)
-
-HTTPRandomPageHelper = Application([
-    # XXX add regexps here
-    (r"/(.*)/(.*)", HTTPRandomPage)
-])
-
diff --git a/oonib/testhelpers/ssl_helpers.py b/oonib/testhelpers/ssl_helpers.py
new file mode 100644
index 0000000..2c07275
--- /dev/null
+++ b/oonib/testhelpers/ssl_helpers.py
@@ -0,0 +1,9 @@
+from twisted.internet import ssl
+from oonib import config
+
+class SSLContext(ssl.DefaultOpenSSLContextFactory):
+    def __init__(self, *args, **kw):
+        ssl.DefaultOpenSSLContextFactory.__init__(self, 
+                config.helpers.ssl.private_key,
+                config.main.ssl.certificate)
+
diff --git a/oonib/testhelpers/tcp_helpers.py b/oonib/testhelpers/tcp_helpers.py
new file mode 100644
index 0000000..57b93d2
--- /dev/null
+++ b/oonib/testhelpers/tcp_helpers.py
@@ -0,0 +1,14 @@
+from twisted.internet.protocol import Protocol, Factory
+
+class TCPEchoProtocol(Protocol):
+    def dataReceived(self, data):
+        self.transport.write(data)
+
+class TCPEchoHelper(Factory):
+    """
+    A very simple echo protocol implementation
+    """
+    protocol = TCPEchoProtocol
+
+
+
diff --git a/ooniprobe.conf b/ooniprobe.conf
index 47f480a..191d809 100644
--- a/ooniprobe.conf
+++ b/ooniprobe.conf
@@ -20,7 +20,8 @@ advanced:
     # XXX change this to point to the directory where you have stored the GeoIP
     # database file. This should be the directory in which OONI is installed
     # /path/to/ooni-probe/data/
-    geoip_data_dir: /usr/share/GeoIP/
+    #geoip_data_dir: /usr/share/GeoIP/
+    geoip_data_dir: /home/x/code/networking/ooni-probe/data/
     debug: true
     threadpool_size: 10
 



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits