[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [oonib/master] Refactor oonib's usage of twisted application API



commit 4d76c781a67b77d0c4115b011ee6257cf933c4ce
Author: aagbsn <aagbsn@xxxxxxxx>
Date:   Wed Aug 27 14:13:48 2014 +0000

    Refactor oonib's usage of twisted application API
---
 bin/oonib            |   16 +++---
 oonib/onion.py       |   96 ++++++++++++++++++++++++++++++++++
 oonib/oonibackend.py |   73 +++++++++++++++++++++-----
 oonib/runner.py      |  140 --------------------------------------------------
 4 files changed, 166 insertions(+), 159 deletions(-)

diff --git a/bin/oonib b/bin/oonib
index 7fb1c35..fcb0c72 100755
--- a/bin/oonib
+++ b/bin/oonib
@@ -12,6 +12,7 @@ sys.path.insert(0, os.path.abspath(os.getcwd()))
 
 from oonib import errors as e
 from oonib.config import config
+from oonib.log import LoggerFactory
 
 try:
     config.load()
@@ -36,9 +37,6 @@ except e.InvalidDeckDirectory, path:
     print "Invalid deck directory: %s" % path
     sys.exit(6)
 
-from oonib import runner
-from oonib.oonibackend import application
-
 if config.main.chroot:
     sys.argv.append('--chroot')
     sys.argv.append(config.chroot)
@@ -46,6 +44,12 @@ if config.main.chroot:
 if not config.main.nodaemon:
     sys.argv.append('-y')
 
-def runApp(config):
-    runner.OBaseRunner(config).run()
-runApp(config.main)
+from oonib.oonibackend import application
+
+from twisted.scripts._twistd_unix import UnixApplicationRunner
+class OBaseRunner(UnixApplicationRunner):
+    temporary_data_dir = None
+    def createOrGetApplication(self):
+        return application
+OBaseRunner.loggerFactory = LoggerFactory
+OBaseRunner(config.main).run()
diff --git a/oonib/onion.py b/oonib/onion.py
new file mode 100644
index 0000000..1950a41
--- /dev/null
+++ b/oonib/onion.py
@@ -0,0 +1,96 @@
+import tempfile
+from oonib import log
+from oonib.config import config
+from twisted.internet import reactor, endpoints
+import os
+
+from random import randint
+import socket
+
+from txtorcon import TCPHiddenServiceEndpoint, TorConfig
+from txtorcon import launch_tor
+
+from txtorcon import __version__ as txtorcon_version
+if tuple(map(int, txtorcon_version.split('.'))) < (0, 9, 0):
+    """
+    Fix for bug in txtorcon versions < 0.9.0 where TCPHiddenServiceEndpoint
+    listens on all interfaces by default.
+    """
+    def create_listener(self, proto):
+        self._update_onion(self.hiddenservice.dir)
+        self.tcp_endpoint = endpoints.TCP4ServerEndpoint(self.reactor,
+                                                         self.listen_port,
+                                                         interface='127.0.0.1')
+        d = self.tcp_endpoint.listen(self.protocolfactory)
+        d.addCallback(self._add_attributes).addErrback(self._retry_local_port)
+        return d
+    TCPHiddenServiceEndpoint._create_listener =  create_listener
+
+def randomFreePort(addr="127.0.0.1"):
+    """
+    Args:
+
+        addr (str): the IP address to attempt to bind to.
+
+    Returns an int representing the free port number at the moment of calling
+
+    Note: there is no guarantee that some other application will attempt to
+    bind to this port once this function has been called.
+    """
+    free = False
+    while not free:
+        port = randint(1024, 65535)
+        s = socket.socket()
+        try:
+            s.bind((addr, port))
+            free = True
+        except:
+            pass
+        s.close()
+    return port
+
+def txSetupFailed(failure):
+    log.err("Setup failed")
+    log.exception(failure)
+
+def startTor(torconfig):
+    def updates(prog, tag, summary):
+        print("%d%%: %s" % (prog, summary))
+
+    if config.main.socks_port:
+        torconfig.SocksPort = config.main.socks_port
+    if config.main.control_port:
+        torconfig.ControlPort = config.main.control_port
+    if config.main.tor2webmode:
+        torconfig.Tor2webMode = 1
+        torconfig.CircuitBuildTimeout = 60
+    if config.main.tor_datadir is None:
+        temporary_data_dir = tempfile.mkdtemp()
+        log.warn("Option 'tor_datadir' in oonib.conf is unspecified!")
+        log.warn("Using %s" % temporary_data_dir)
+        torconfig.DataDirectory = temporary_data_dir
+    else:
+        if os.path.exists(config.main.tor_datadir):
+            torconfig.DataDirectory = os.path.abspath(config.main.tor_datadir)
+        else:
+            raise Exception
+    torconfig.save()
+    if not hasattr(torconfig, 'ControlPort'):
+        control_port = int(randomFreePort())
+        torconfig.ControlPort = control_port
+        config.main.control_port = control_port
+
+    if not hasattr(torconfig, 'SocksPort'):
+        socks_port = int(randomFreePort())
+        torconfig.SocksPort = socks_port
+        config.main.socks_port = socks_port
+
+    torconfig.save()
+
+    if config.main.tor_binary is not None:
+        d = launch_tor(torconfig, reactor,
+                       tor_binary=config.main.tor_binary,
+                       progress_updates=updates)
+    else:
+        d = launch_tor(torconfig, reactor, progress_updates=updates)
+    return d
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 2a9fa7d..ca51f45 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -7,13 +7,19 @@
 # In here we start all the test helpers that are required by ooniprobe and
 # start the report collector
 
-from twisted.application import internet, service
-from twisted.names import dns
-
+from oonib.api import ooniBackend, ooniBouncer
+from oonib.config import config
+from oonib.onion import startTor
 from oonib.testhelpers import dns_helpers, ssl_helpers
 from oonib.testhelpers import http_helpers, tcp_helpers
 
-from oonib.config import config
+import os
+
+from twisted.application import internet, service
+from twisted.internet import reactor
+from twisted.names import dns
+
+from txtorcon import TCPHiddenServiceEndpoint, TorConfig
 
 if config.main.uid and config.main.gid:
     application = service.Application('oonibackend', uid=config.main.uid,
@@ -21,28 +27,28 @@ if config.main.uid and config.main.gid:
 else:
     application = service.Application('oonibackend')
 
-serviceCollection = service.IServiceCollection(application)
+multiService = service.MultiService()
 
 if config.helpers['ssl'].port:
     print "Starting SSL helper on %s" % config.helpers['ssl'].port
     ssl_helper = internet.SSLServer(int(config.helpers['ssl'].port),
                                     http_helpers.HTTPReturnJSONHeadersHelper(),
                                     ssl_helpers.SSLContext(config))
-    ssl_helper.setServiceParent(serviceCollection)
+    multiService.addService(ssl_helper)
 
 # Start the DNS Server related services
 if config.helpers['dns'].tcp_port:
     print "Starting TCP DNS Helper on %s" % config.helpers['dns'].tcp_port
     tcp_dns_helper = internet.TCPServer(int(config.helpers['dns'].tcp_port),
                                         dns_helpers.DNSTestHelper())
-    tcp_dns_helper.setServiceParent(serviceCollection)
+    multiService.addService(tcp_dns_helper)
 
 if config.helpers['dns'].udp_port:
     print "Starting UDP DNS Helper on %s" % config.helpers['dns'].udp_port
     udp_dns_factory = dns.DNSDatagramProtocol(dns_helpers.DNSTestHelper())
     udp_dns_helper = internet.UDPServer(int(config.helpers['dns'].udp_port),
                                         udp_dns_factory)
-    udp_dns_helper.setServiceParent(serviceCollection)
+    multiService.addService(udp_dns_helper)
 
 if config.helpers['dns_discovery'].udp_port:
     print ("Starting UDP DNS Discovery Helper on %s" %
@@ -51,14 +57,14 @@ if config.helpers['dns_discovery'].udp_port:
                                            dns.DNSDatagramProtocol(
                                                dns_helpers.DNSResolverDiscovery()
                                            ))
-    udp_dns_discovery.setServiceParent(serviceCollection)
+    multiService.addService(udp_dns_discovery)
 
 if config.helpers['dns_discovery'].tcp_port:
     print ("Starting TCP DNS Discovery Helper on %s" %
            config.helpers['dns_discovery'].tcp_port)
     tcp_dns_discovery = internet.TCPServer(int(config.helpers['dns_discovery'].tcp_port),
                                            dns_helpers.DNSResolverDiscovery())
-    tcp_dns_discovery.setServiceParent(serviceCollection)
+    multiService.addService(tcp_dns_discovery)
 
 
 # XXX this needs to be ported
@@ -67,18 +73,59 @@ if config.helpers['daphn3'].port:
     print "Starting Daphn3 helper on %s" % config.helpers['daphn3'].port
     daphn3_helper = internet.TCPServer(int(config.helpers['daphn3'].port),
                                        tcp_helpers.Daphn3Server())
-    daphn3_helper.setServiceParent(serviceCollection)
+    multiService.addService(daphn3_helper)
 
 
 if config.helpers['tcp-echo'].port:
     print "Starting TCP echo helper on %s" % config.helpers['tcp-echo'].port
     tcp_echo_helper = internet.TCPServer(int(config.helpers['tcp-echo'].port),
                                          tcp_helpers.TCPEchoHelper())
-    tcp_echo_helper.setServiceParent(serviceCollection)
+    multiService.addService(tcp_echo_helper)
 
 if config.helpers['http-return-json-headers'].port:
     print "Starting HTTP return request helper on %s" % config.helpers['http-return-json-headers'].port
     http_return_request_helper = internet.TCPServer(
         int(config.helpers['http-return-json-headers'].port),
         http_helpers.HTTPReturnJSONHeadersHelper())
-    http_return_request_helper.setServiceParent(serviceCollection)
+multiService.addService(http_return_request_helper)
+
+# add the tor collector service here
+if config.main.tor_hidden_service:
+    torconfig = TorConfig()
+    d = startTor(torconfig)
+
+    def addCollector(torControlProtocol):
+        data_dir = os.path.join(torconfig.DataDirectory, 'collector')
+        collector_service = internet.StreamServerEndpointService(
+                TCPHiddenServiceEndpoint(reactor,
+                    torconfig, 80,
+                    hidden_service_dir=data_dir),
+                ooniBackend)
+        multiService.addService(collector_service)
+        collector_service.startService()
+        return torControlProtocol
+
+    d.addCallback(addCollector)
+
+    if ooniBouncer:
+        def addBouncer(torControlProtocol):
+            data_dir = os.path.join(torconfig.DataDirectory, 'bouncer')
+
+            bouncer_service = internet.StreamServerEndpointService(
+                    TCPHiddenServiceEndpoint(reactor,
+                        torconfig, 80,
+                        hidden_service_dir=data_dir),
+                    ooniBouncer)
+            multiService.addService(bouncer_service)
+            bouncer_service.startService()
+            return torControlProtocol
+
+        d.addCallback(addBouncer)
+else:
+    if ooniBouncer:
+        bouncer_service = internet.TCPServer(8888, ooniBouncer, interface="127.0.0.1")
+        multiService.addService(bouncer_service)
+        bouncer_service.startService()
+    collector_service = internet.TCPServer(8889, ooniBackend, interface="127.0.0.1")
+    multiService.addService(collector_service)
+    collector_service.startService()
diff --git a/oonib/runner.py b/oonib/runner.py
deleted file mode 100644
index 47ddc35..0000000
--- a/oonib/runner.py
+++ /dev/null
@@ -1,140 +0,0 @@
-"""
-In here we define a runner for the oonib backend system.
-"""
-
-from __future__ import print_function
-
-from distutils.version import LooseVersion
-import tempfile
-import os
-
-from shutil import rmtree
-
-from twisted.internet import reactor, endpoints
-from twisted.python.runtime import platformType
-
-from txtorcon import TCPHiddenServiceEndpoint, TorConfig
-from txtorcon import launch_tor
-
-from oonib.api import ooniBackend, ooniBouncer
-from oonib.config import config
-
-from oonib import oonibackend
-from oonib import log
-
-from txtorcon import __version__ as txtorcon_version
-if tuple(map(int, txtorcon_version.split('.'))) < (0, 9, 0):
-    """
-    Fix for bug in txtorcon versions < 0.9.0 where TCPHiddenServiceEndpoint
-    listens on all interfaces by default.
-    """
-    def create_listener(self, proto):
-        self._update_onion(self.hiddenservice.dir)
-        self.tcp_endpoint = endpoints.TCP4ServerEndpoint(self.reactor,
-                                                         self.listen_port,
-                                                         interface='127.0.0.1')
-        d = self.tcp_endpoint.listen(self.protocolfactory)
-        d.addCallback(self._add_attributes).addErrback(self._retry_local_port)
-        return d
-    TCPHiddenServiceEndpoint._create_listener =  create_listener
-
-class OBaseRunner(object):
-    pass
-
-if platformType == "win32":
-    from twisted.scripts._twistw import WindowsApplicationRunner
-
-    OBaseRunner = WindowsApplicationRunner
-    # XXX Currently we don't support windows for starting a Tor Hidden Service
-    log.warn(
-        "Apologies! We don't support starting a Tor Hidden Service on Windows.")
-
-else:
-    from twisted.scripts._twistd_unix import UnixApplicationRunner
-    class OBaseRunner(UnixApplicationRunner):
-        temporary_data_dir = None
-
-        def txSetupFailed(self, failure):
-            log.err("Setup failed")
-            log.exception(failure)
-
-        def setupHSEndpoint(self, tor_process_protocol, torconfig, endpoint):
-            endpointName = endpoint.settings['name']
-
-            def setup_complete(port):
-                if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'):
-                    onion_uri = port.address.onion_uri
-                else:
-                    onion_uri = port.onion_uri
-                print("Exposed %s Tor hidden service "
-                      "on httpo://%s" % (endpointName, onion_uri))
-
-            public_port = 80
-            data_dir = os.path.join(torconfig.DataDirectory, endpointName)
-            if LooseVersion(txtorcon_version) >= LooseVersion('0.10.0'):
-                hs_endpoint = TCPHiddenServiceEndpoint(reactor,
-                                                       torconfig,
-                                                       public_port,
-                                                       hidden_service_dir=data_dir)
-            else:
-                hs_endpoint = TCPHiddenServiceEndpoint(reactor,
-                                                       torconfig,
-                                                       public_port,
-                                                       data_dir=data_dir)
-            d = hs_endpoint.listen(endpoint)
-            d.addCallback(setup_complete)
-            d.addErrback(self.txSetupFailed)
-            return d
-
-        def startTor(self, torconfig):
-            def updates(prog, tag, summary):
-                print("%d%%: %s" % (prog, summary))
-
-            torconfig.SocksPort = config.main.socks_port
-            if config.main.tor2webmode:
-                torconfig.Tor2webMode = 1
-                torconfig.CircuitBuildTimeout = 60
-            if config.main.tor_datadir is None:
-                self.temporary_data_dir = tempfile.mkdtemp()
-                log.warn("Option 'tor_datadir' in oonib.conf is unspecified!")
-                log.warn("Using %s" % self.temporary_data_dir)
-                torconfig.DataDirectory = self.temporary_data_dir
-            else:
-                torconfig.DataDirectory = config.main.tor_datadir
-            torconfig.save()
-            if config.main.tor_binary is not None:
-                d = launch_tor(torconfig, reactor,
-                               tor_binary=config.main.tor_binary,
-                               progress_updates=updates)
-            else:
-                d = launch_tor(torconfig, reactor, progress_updates=updates)
-            return d
-
-        def postApplication(self):
-            """After the application is created, start the application and run
-            the reactor. After the reactor stops, clean up PID files and such.
-            """
-            self.startApplication(self.application)
-            # This is our addition. The rest is taken from
-            # twisted/scripts/_twistd_unix.py 12.2.0
-            if config.main.tor_hidden_service:
-                torconfig = TorConfig()
-                d = self.startTor(torconfig)
-                d.addCallback(self.setupHSEndpoint, torconfig, ooniBackend)
-                if ooniBouncer:
-                    d.addCallback(self.setupHSEndpoint, torconfig, ooniBouncer)
-            else:
-                if ooniBouncer:
-                    reactor.listenTCP(8888, ooniBouncer, interface="127.0.0.1")
-                reactor.listenTCP(8889, ooniBackend, interface="127.0.0.1")
-            self.startReactor(None, self.oldstdout, self.oldstderr)
-            self.removePID(self.config['pidfile'])
-            if self.temporary_data_dir:
-                log.msg("Removing temporary directory: %s"
-                        % self.temporary_data_dir)
-                rmtree(self.temporary_data_dir, onerror=log.err)
-
-        def createOrGetApplication(self):
-            return oonibackend.application
-
-OBaseRunner.loggerFactory = log.LoggerFactory



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits