[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/develop] Improve configuration management



commit c7f65ae7df71c469f9cf5db126f12bf1c11c49da
Author: Arturo Filastò <art@xxxxxxxxx>
Date:   Tue Apr 23 16:59:24 2013 +0200

    Improve configuration management
    
    * Install ooniprobe settings in the users home directory (~/.ooni)
    * Define the paths for ooniprobe data directories
---
 ooni/__init__.py            |    2 -
 ooni/api/spec.py            |    2 +-
 ooni/config.py              |   98 -------------------------------------------
 ooni/director.py            |   15 ++++---
 ooni/geoip.py               |    3 +-
 ooni/managers.py            |   18 +++++---
 ooni/nettest.py             |    2 +-
 ooni/oonicli.py             |   11 ++++-
 ooni/oonid.py               |    2 +-
 ooni/reporter.py            |    2 +-
 ooni/settings.py            |   92 ++++++++++++++++++++++++++++++++++++++++
 ooni/tasks.py               |   10 ++---
 ooni/templates/httpt.py     |    2 +-
 ooni/templates/scapyt.py    |    2 +-
 ooni/tests/mocks.py         |    2 +-
 ooni/tests/test_managers.py |    8 ++++
 ooni/tests/test_nettest.py  |    5 ++-
 ooni/utils/geodata.py       |    2 +-
 ooni/utils/log.py           |    2 +-
 ooni/utils/txscapy.py       |    2 +-
 20 files changed, 149 insertions(+), 133 deletions(-)

diff --git a/ooni/__init__.py b/ooni/__init__.py
index cc4bf03..815e16e 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,5 @@
 # -*- encoding: utf-8 -*-
 
-from . import config
-from . import kit
 from . import nettest
 from . import oonicli
 from . import reporter
diff --git a/ooni/api/spec.py b/ooni/api/spec.py
index af238f4..5b538b2 100644
--- a/ooni/api/spec.py
+++ b/ooni/api/spec.py
@@ -5,7 +5,7 @@ import types
 
 from cyclone import web, escape
 
-from ooni import config
+from ooni.settings import config
 
 class InvalidInputFilename(Exception):
     pass
diff --git a/ooni/config.py b/ooni/config.py
deleted file mode 100644
index 5aeb49d..0000000
--- a/ooni/config.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import os
-import yaml
-
-from twisted.internet import reactor, threads, defer
-
-from ooni import otime
-from ooni.utils import Storage
-
-class TestFilenameNotSet(Exception):
-    pass
-
-def get_root_path():
-    this_directory = os.path.dirname(__file__)
-    root = os.path.join(this_directory, '..')
-    root = os.path.abspath(root)
-    return root
-
-def createConfigFile():
-    """
-    XXX implement me
-    """
-    sample_config_file = os.path.join(get_root_path(), 'ooniprobe.conf.sample')
-
-def generatePcapFilename():
-    if cmd_line_options['pcapfile']:
-        reports.pcap = cmd_line_options['pcapfile']
-    else:
-        if cmd_line_options['test']:
-            test_filename = os.path.basename(cmd_line_options['test'])
-        else:
-            test_filename = os.path.basename(cmd_line_options['testdeck'])
-
-        test_name = '.'.join(test_filename.split(".")[:-1])
-        frm_str = "report_%s_"+otime.timestamp()+".%s"
-        reports.pcap = frm_str % (test_name, "pcap")
-
-class ConfigurationSetting(Storage):
-    def __init__(self, key):
-        config_file = os.path.join(get_root_path(), 'ooniprobe.conf')
-        try:
-            f = open(config_file)
-        except IOError:
-            createConfigFile()
-            raise Exception("Unable to open config file. "\
-                        "Copy ooniprobe.conf.sample to ooniprobe.conf")
-
-        config_file_contents = '\n'.join(f.readlines())
-        configuration = yaml.safe_load(config_file_contents)
-
-        try:
-            for k, v in configuration[key].items():
-                self[k] = v
-        except AttributeError:
-            pass
-
-basic = ConfigurationSetting('basic')
-advanced = ConfigurationSetting('advanced')
-privacy = ConfigurationSetting('privacy')
-tor = ConfigurationSetting('tor')
-
-data_directory = os.path.join(get_root_path(), 'data')
-nettest_directory = os.path.join(get_root_path(), 'nettests')
-inputs_directory = os.path.join(get_root_path(), 'inputs')
-
-reports = Storage()
-state = Storage()
-scapyFactory = None
-stateDict = None
-
-# XXX refactor this to use a database
-resume_lock = defer.DeferredLock()
-
-cmd_line_options = None
-resume_filename = None
-
-# XXX-Twisted this is used to check if we have started the reactor or not. It
-# is necessary because if the tests are already concluded because we have
-# resumed a test session then it will call reactor.run() even though there is
-# no condition that will ever stop it.
-# There should be a more twisted way of doing this.
-start_reactor = True
-tor_state = None
-tor_control = None
-config_file = None
-sample_config_file = None
-# This is used to store the probes IP address obtained via Tor
-probe_ip = None
-# This is used to keep track of the state of the sniffer
-sniffer_running = None
-
-logging = True
-
-if not resume_filename:
-    resume_filename = os.path.join(get_root_path(), 'ooniprobe.resume')
-    try:
-        with open(resume_filename) as f: pass
-    except IOError as e:
-        with open(resume_filename, 'w+') as f: pass
diff --git a/ooni/director.py b/ooni/director.py
index 5f23668..1ef878c 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -3,13 +3,13 @@ import sys
 import os
 import re
 
-from ooni import config
 from ooni import geoip
 from ooni.managers import ReportEntryManager, MeasurementManager
 from ooni.reporter import Report
 from ooni.utils import log, checkForRoot
 from ooni.utils.net import randomFreePort
-from ooni.nettest import NetTest
+from ooni.nettest import NetTest, getNetTestInformation
+from ooni.settings import config
 from ooni import errors
 
 from txtorcon import TorConfig
@@ -85,6 +85,10 @@ class Director(object):
 
         self.torControlProtocol = None
 
+        # This deferred is fired once all the measurements and their reporting
+        # tasks are completed.
+        self.allTestsDone = defer.Deferred()
+
     def getNetTests(self):
         nettests = {}
         def is_nettest(filename):
@@ -108,16 +112,12 @@ class Director(object):
 
         return nettests
 
-        # This deferred is fired once all the measurements and their reporting
-        # tasks are completed.
-        self.allTestsDone = defer.Deferred()
-
     @defer.inlineCallbacks
     def start(self):
         if config.privacy.includepcap:
             log.msg("Starting")
             if not config.reports.pcap:
-                config.reports.pcap = config.generatePcapFilename()
+                config.generate_pcap_filename()
             self.startSniffing()
 
         if config.advanced.start_tor:
@@ -324,7 +324,6 @@ class Director(object):
         log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)
 
         d = launch_tor(tor_config, reactor,
-                tor_binary=config.advanced.tor_binary,
                 progress_updates=updates)
         d.addCallback(setup_complete)
         d.addErrback(setup_failed)
diff --git a/ooni/geoip.py b/ooni/geoip.py
index 1e6f6e9..c1987fc 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -7,7 +7,8 @@ from ooni.utils.net import userAgents, BodyReceiver
 from twisted.internet import reactor, defer, protocol
 
 from ooni.utils import log, net, checkForRoot
-from ooni import config, errors
+from ooni.settings import config
+from ooni import errors
 
 try:
     from pygeoip import GeoIP
diff --git a/ooni/managers.py b/ooni/managers.py
index 9f8366f..ff7c2f2 100644
--- a/ooni/managers.py
+++ b/ooni/managers.py
@@ -2,7 +2,7 @@ import itertools
 
 from twisted.internet import defer
 from ooni.utils import log
-from ooni import config
+from ooni.settings import config
 
 def makeIterable(item):
     """
@@ -141,8 +141,12 @@ class MeasurementManager(TaskManager):
     NetTest on the contrary is aware of the typology of measurements that it is
     dispatching as they are logically grouped by test file.
     """
-    retries = config.advanced.measurement_retries
-    concurrency = config.advanced.measurement_concurrency
+    def __init__(self):
+        if config.advanced.measurement_retries:
+            self.retries = config.advanced.measurement_retries
+        if config.advanced.measurement_concurrency:
+            self.concurrency = config.advanced.measurement_concurrency
+        super(MeasurementManager, self).__init__()
 
     def succeeded(self, result, measurement):
         log.debug("Successfully performed measurement %s" % measurement)
@@ -152,8 +156,12 @@ class MeasurementManager(TaskManager):
         pass
 
 class ReportEntryManager(TaskManager):
-    retries = config.advanced.reporting_retries
-    concurrency = config.advanced.reporting_concurrency
+    def __init__(self):
+        if config.advanced.reporting_retries:
+            self.retries = config.advanced.reporting_retries
+        if config.advanced.reporting_concurrency:
+            self.concurrency = config.advanced.reporting_concurrency
+        super(ReportEntryManager, self).__init__()
 
     def succeeded(self, result, task):
         log.debug("Successfully performed report %s" % task)
diff --git a/ooni/nettest.py b/ooni/nettest.py
index d9bc94d..0d0e889 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -8,8 +8,8 @@ from twisted.python import usage, reflect
 from ooni import geoip
 from ooni.tasks import Measurement
 from ooni.utils import log, checkForRoot, geodata
-from ooni import config
 from ooni import otime
+from ooni.settings import config
 
 from ooni import errors as e
 
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index b5e8e27..40453b1 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -12,7 +12,7 @@ from twisted.python.util import spewer
 
 from ooni import errors
 
-from ooni import config
+from ooni.settings import config
 from ooni.director import Director
 from ooni.reporter import YAMLReporter, OONIBReporter
 from ooni.nettest import NetTestLoader, MissingRequiredOption
@@ -40,6 +40,10 @@ class Options(usage.Options):
                      ["logfile", "l", None, "log file name"],
                      ["pcapfile", "O", None, "pcap file name"],
                      ["parallelism", "p", "10", "input parallelism"],
+                     ["configfile", "f", None,
+                         "Specify a path to the ooniprobe configuration file"],
+                     ["datadir", "d", None,
+                         "Specify a path to the ooniprobe data directory"]
                      ]
 
     compData = usage.Completions(
@@ -100,8 +104,11 @@ def runWithDirector():
     test!
     """
     global_options = parseOptions()
-    log.start(global_options['logfile'])
+    config.global_options = global_options
+    config.set_paths()
+    config.read_config_file()
 
+    log.start(global_options['logfile'])
     # contains (test_cases, options, cmd_line_options)
     test_list = []
     if global_options['no-collector']:
diff --git a/ooni/oonid.py b/ooni/oonid.py
index dde768e..cc71d47 100644
--- a/ooni/oonid.py
+++ b/ooni/oonid.py
@@ -4,7 +4,7 @@ import random
 from twisted.application import service, internet
 from twisted.web import static, server
 
-from ooni import config
+from ooni.settings import config
 from ooni.api.spec import oonidApplication
 from ooni.director import Director
 from ooni.reporter import YAMLReporter, OONIBReporter
diff --git a/ooni/reporter.py b/ooni/reporter.py
index b04b46b..109eccf 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -32,7 +32,7 @@ from ooni import otime
 from ooni.utils import geodata, pushFilenameStack
 from ooni.utils.net import BodyReceiver, StringProducer, userAgents
 
-from ooni import config
+from ooni.settings import config
 
 from ooni.tasks import ReportEntry, TaskTimedOut
 
diff --git a/ooni/settings.py b/ooni/settings.py
new file mode 100644
index 0000000..acb7502
--- /dev/null
+++ b/ooni/settings.py
@@ -0,0 +1,92 @@
+import os
+import yaml
+from shutil import copyfile
+from os.path import abspath, expanduser
+
+from twisted.internet import reactor, threads, defer
+
+from ooni import otime
+from ooni.utils import Storage
+
+class OConfig(object):
+    def __init__(self):
+        self.global_options = {}
+        self.reports = Storage()
+        self.scapyFactory = None
+        self.tor_state = None
+        # This is used to store the probes IP address obtained via Tor
+        self.probe_ip = None
+        # This is used to keep track of the state of the sniffer
+        self.sniffer_running = None
+        self.logging = True
+        self.basic = Storage()
+        self.advanced = Storage()
+        self.tor = Storage()
+        self.privacy = Storage()
+        self.set_paths()
+        self.initialize_ooni_home()
+
+    def set_paths(self):
+        if self.global_options.get('datadir'):
+            self.data_directory = abspath(expanduser(self.global_options['datadir']))
+        else:
+            self.data_directory = '/usr/share/ooni/'
+        self.nettest_directory = os.path.join(self.data_directory, 'nettests')
+
+        self.ooni_home = os.path.join(expanduser('~'), '.ooni')
+        self.inputs_directory = os.path.join(self.ooni_home, 'inputs')
+
+        if self.global_options.get('configfile'):
+            config_file = global_options['configfile']
+        else:
+            config_file = os.path.join('~', '.ooni', 'ooniprobe.conf')
+        self.config_file = expanduser(config_file)
+
+    def initialize_ooni_home(self):
+        if not os.path.isdir(self.ooni_home):
+            print "Ooni home directory does not exist."
+            print "Creating it in '%s'." % self.ooni_home
+            os.mkdir(self.ooni_home)
+            os.mkdir(self.inputs_directory)
+
+    def _create_config_file(self):
+        sample_config_file = os.path.join(self.data_directory,
+                                          'ooniprobe.conf.sample')
+        target_config_file = os.path.join(self.ooni_home,
+                                          'ooniprobe.conf')
+        print "Creating it for you in '%s'." % target_config_file
+        copyfile(sample_config_file, target_config_file)
+
+    def read_config_file(self):
+        try:
+            with open(self.config_file) as f: pass
+        except IOError:
+            print "Configuration file does not exist."
+            self._create_config_file()
+            self.read_config_file()
+
+        with open(self.config_file) as f:
+            config_file_contents = '\n'.join(f.readlines())
+            configuration = yaml.safe_load(config_file_contents)
+
+            for setting in ['basic', 'advanced', 'privacy', 'tor']:
+                try:
+                    for k, v in configuration[setting].items():
+                        getattr(self, setting)[k] = v
+                except AttributeError:
+                    pass
+
+    def generate_pcap_filename():
+        if self.global_options.get('pcapfile'):
+            self.reports.pcap = self.global_options['pcapfile']
+        else:
+            if self.global_options.get('test'):
+                test_filename = os.path.basename(self.global_options['test'])
+            else:
+                test_filename = os.path.basename(self.global_options['testdeck'])
+
+            test_name = '.'.join(test_filename.split(".")[:-1])
+            frm_str = "report_%s_"+otime.timestamp()+".%s"
+            self.reports.pcap = frm_str % (test_name, "pcap")
+
+config = OConfig()
diff --git a/ooni/tasks.py b/ooni/tasks.py
index 829d11e..f686a9c 100644
--- a/ooni/tasks.py
+++ b/ooni/tasks.py
@@ -1,6 +1,6 @@
 import time
 
-from ooni import config
+from ooni.settings import config
 from twisted.internet import defer, reactor
 
 class BaseTask(object):
@@ -91,8 +91,6 @@ class TaskWithTimeout(BaseTask):
         return BaseTask.start(self)
 
 class Measurement(TaskWithTimeout):
-    timeout = config.advanced.measurement_timeout
-
     def __init__(self, test_class, test_method, test_input):
         """
         test_class:
@@ -117,6 +115,8 @@ class Measurement(TaskWithTimeout):
 
         self.netTestMethod = getattr(self.testInstance, test_method)
 
+        if config.advanced.measurement_timeout:
+            self.timeout = config.advanced.measurement_timeout
         TaskWithTimeout.__init__(self)
 
     def succeeded(self, result):
@@ -130,12 +130,12 @@ class Measurement(TaskWithTimeout):
         return d
 
 class ReportEntry(TaskWithTimeout):
-    timeout = config.advanced.reporting_timeout
-
     def __init__(self, reporter, measurement):
         self.reporter = reporter
         self.measurement = measurement
 
+        if config.advanced.reporting_timeout:
+            self.timeout = config.advanced.reporting_timeout
         TaskWithTimeout.__init__(self)
 
     def run(self):
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index e8891c7..0bca5df 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -13,7 +13,7 @@ from twisted.web._newclient import Request, Response, ResponseNeverReceived
 
 from ooni.nettest import NetTestCase
 from ooni.utils import log
-from ooni import config
+from ooni.settings import config
 
 from ooni.utils.net import BodyReceiver, StringProducer, userAgents
 
diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py
index d5d6564..fdc5a24 100644
--- a/ooni/templates/scapyt.py
+++ b/ooni/templates/scapyt.py
@@ -8,7 +8,7 @@ from scapy.all import send, sr, IP, TCP, config
 from ooni.reporter import createPacketReport
 from ooni.nettest import NetTestCase
 from ooni.utils import log
-from ooni import config
+from ooni.settings import config
 
 from ooni.utils.txscapy import ScapySender, getDefaultIface, ScapyFactory
 from ooni.utils.txscapy import hasRawSocketPermission
diff --git a/ooni/tests/mocks.py b/ooni/tests/mocks.py
index 4c4a015..f849344 100644
--- a/ooni/tests/mocks.py
+++ b/ooni/tests/mocks.py
@@ -1,7 +1,7 @@
 from twisted.python import failure
 from twisted.internet import defer
 
-from ooni import config
+from ooni.settings import config
 from ooni.tasks import BaseTask, TaskWithTimeout
 from ooni.nettest import NetTest
 from ooni.managers import TaskManager
diff --git a/ooni/tests/test_managers.py b/ooni/tests/test_managers.py
index e2af7b3..c290155 100644
--- a/ooni/tests/test_managers.py
+++ b/ooni/tests/test_managers.py
@@ -1,3 +1,5 @@
+import os
+
 from twisted.trial import unittest
 from twisted.python import failure
 from twisted.internet import defer, task
@@ -11,6 +13,8 @@ from ooni.tests.mocks import MockTimeoutOnceTask, MockFailTaskWithTimeout
 from ooni.tests.mocks import MockTaskManager, mockFailure, MockDirector
 from ooni.tests.mocks import MockNetTest, MockMeasurement, MockSuccessMeasurement
 from ooni.tests.mocks import MockFailMeasurement, MockFailOnceMeasurement
+from ooni.settings import config
+
 
 class TestTaskManager(unittest.TestCase):
     timeout = 1
@@ -22,6 +26,10 @@ class TestTaskManager(unittest.TestCase):
         self.measurementManager.start()
 
         self.clock = task.Clock()
+        data_dir = os.path.dirname(os.path.abspath(__file__))
+        data_dir = os.path.join(data_dir, '..', '..', 'data')
+        config.global_options['datadir'] = data_dir
+        config.set_paths()
 
     def schedule_successful_tasks(self, task_type, number=1):
         all_done = []
diff --git a/ooni/tests/test_nettest.py b/ooni/tests/test_nettest.py
index 3b59cc4..77b8a1c 100644
--- a/ooni/tests/test_nettest.py
+++ b/ooni/tests/test_nettest.py
@@ -11,7 +11,6 @@ from ooni.nettest import NetTestLoader, FailureToLoadNetTest
 from ooni.tasks import BaseTask
 
 from ooni.director import Director
-
 from ooni.managers import TaskManager
 
 from ooni.tests.mocks import MockMeasurement, MockMeasurementFailOnce
@@ -100,6 +99,9 @@ class TestNetTest(unittest.TestCase):
             for i in range(10):
                 f.write("%s\n" % i)
 
+        from ooni.settings import config
+        config.read_config_file()
+
     def assertCallable(self, thing):
         self.assertIn('__call__', dir(thing))
 
@@ -226,7 +228,6 @@ class TestNetTest(unittest.TestCase):
 
         @d.addCallback
         def complete(result):
-            print "IN here y0"
             self.assertEqual(result, None)
             self.assertEqual(director.successfulMeasurements, 20)
 
diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py
index 2acfdb0..c8a9a3a 100644
--- a/ooni/utils/geodata.py
+++ b/ooni/utils/geodata.py
@@ -5,8 +5,8 @@ from twisted.web.client import Agent
 from twisted.internet import reactor, defer, protocol
 
 from ooni.utils import log, net
-from ooni import config
 from ooni.errors import GeoIPDataFilesNotFound
+from ooni.settings import config
 
 try:
     import pygeoip
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 141116e..067d6a6 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -9,7 +9,7 @@ from twisted.python.failure import Failure
 from twisted.python.logfile import DailyLogFile
 
 from ooni import otime
-from ooni import config
+from ooni.settings import config
 
 ## Get rid of the annoying "No route found for
 ## IPv6 destination warnings":
diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py
index 80dd1c2..e02de60 100644
--- a/ooni/utils/txscapy.py
+++ b/ooni/utils/txscapy.py
@@ -12,7 +12,7 @@ from zope.interface import implements
 from scapy.config import conf
 
 from ooni.utils import log
-from ooni import config
+from ooni.settings import config
 
 class LibraryNotInstalledError(Exception):
     pass



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits