[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Add support for parsing multiple config files



commit 31bcae623b8a621aa941f284e67438168243c44a
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Fri Sep 2 15:00:48 2016 +0200

    Add support for parsing multiple config files
    
    * Configuration files are parsed in a certain order and they override the
      default behavior.
---
 Vagrantfile                     |   5 +
 ooni/__init__.py                |   2 +-
 ooni/scripts/ooniprobe_agent.py |   1 +
 ooni/settings.py                | 125 ++++++++++++++-----
 ooni/tests/test_settings.py     |  61 ++++++++++
 ooni/ui/cli.py                  |   4 +-
 ooni/ui/web/client/index.html   |   2 +-
 setup.py                        | 262 ++++++++++++++++++----------------------
 8 files changed, 285 insertions(+), 177 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 71f7c7a..8ff0bbd 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -81,4 +81,9 @@ Vagrant.configure("2") do |config|
     end
   end
 
+  config.vm.define "testing" do |testing|
+    testing.vm.network "forwarded_port", guest: 8842, host: 8142
+    testing.vm.synced_folder ".", "/data/ooni-probe"
+  end
+
 end
diff --git a/ooni/__init__.py b/ooni/__init__.py
index 9eecb67..d4b6f95 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0-alpha.2"
+__version__ = "2.0.0-alpha.3"
 
 __all__ = [
     'agent',
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index f56f2df..3eb1d22 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -58,6 +58,7 @@ def start_agent(options=None):
     #  a null log observer
     twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull',
                    '--umask', '022']
+
     twistd_config = OoniprobeTwistdConfig()
     if options is not None:
         twistd_args.extend(options.twistd_args)
diff --git a/ooni/settings.py b/ooni/settings.py
index a0ef0a5..66bd212 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -3,6 +3,7 @@ import sys
 import yaml
 import errno
 import getpass
+from pkg_resources import parse_version
 from ConfigParser import SafeConfigParser
 
 from twisted.internet import defer, reactor
@@ -10,6 +11,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
 
 from os.path import abspath, expanduser
 
+from ooni import __version__ as ooniprobe_version
 from ooni.utils import Storage, log, get_ooni_root
 
 CONFIG_FILE_TEMPLATE = """\
@@ -19,8 +21,8 @@ CONFIG_FILE_TEMPLATE = """\
 
 basic:
     # Where OONIProbe should be writing it's log file
-    logfile: {logfile}
-    loglevel: WARNING
+    # logfile: {logfile}
+    # loglevel: WARNING
     # The maximum amount of data to store on disk. Once the quota is reached,
     # we will start deleting older reports.
     # measurement_quota: 1G
@@ -191,6 +193,37 @@ elif os.path.isfile(_SETTINGS_INI):
     if _ETC_PATH is not None:
         ETC_PATH = _ETC_PATH
 
+
+def _load_config_files_with_defaults(config_files, defaults):
+    """
+    This takes care of reading the config files in reverse order (the first
+    item will have priority over the last element) and produce a
+    configuration that includes ONLY the options inside of the defaults
+    dictionary.
+
+    :param config_files: a list of configuration file paths
+    :param defaults: the default values for the configuration file
+    :return: a configuration that is the result of reading the config files
+    and joining it with the default options.
+    """
+    config_from_files = {}
+    configuration = {}
+    for config_file_path in reversed(config_files):
+        if not os.path.exists(config_file_path):
+            continue
+        with open(config_file_path) as in_file:
+            c = yaml.safe_load(in_file)
+        config_from_files.update(c)
+
+    for category in defaults.keys():
+        configuration[category] = {}
+        for k, v in defaults[category].items():
+            try:
+                configuration[category][k] = config_from_files[category][k]
+            except KeyError:
+                configuration[category][k] = defaults[category][k]
+    return configuration
+
 class OConfig(object):
     _custom_home = None
 
@@ -211,6 +244,10 @@ class OConfig(object):
         self.tor = Storage()
         self.privacy = Storage()
 
+        # In here we store the configuration files ordered by priority.
+        # First configuration file takes priority over the others.
+        self.config_files = []
+
         self.set_paths()
 
     def is_initialized(self):
@@ -225,6 +262,31 @@ class OConfig(object):
         with open(initialized_path, 'w+'): pass
 
     @property
+    def last_run_version(self):
+        """
+        :return: Version identifying the last run version of ooniprobe.
+        """
+        last_run_version_path = os.path.join(
+            self.running_path, "last_run_version"
+        )
+        if not os.path.exists(last_run_version_path):
+            return parse_version("0")
+        with open(last_run_version_path) as in_file:
+            last_run_version = in_file.read()
+        return parse_version(last_run_version)
+
+    @property
+    def current_version(self):
+        return parse_version(ooniprobe_version)
+
+    def set_last_run_version(self):
+        last_run_version_path = os.path.join(
+            self.running_path, "last_run_version"
+        )
+        with open(last_run_version_path, "w") as out_file:
+            out_file.write(ooniprobe_version)
+
+    @property
     def running_path(self):
         """
         This is the directory used to store state application data.
@@ -266,6 +328,10 @@ class OConfig(object):
         return VAR_LIB_PATH
 
     @property
+    def user_config_file_path(self):
+        return os.path.join(self.running_path, 'ooniprobe.conf')
+
+    @property
     def ooni_home(self):
         home = expanduser('~'+self.current_user)
         if os.getenv("HOME"):
@@ -283,7 +349,8 @@ class OConfig(object):
 
     def set_paths(self):
         self.nettest_directory = os.path.join(OONIPROBE_ROOT, 'nettests')
-        self.web_ui_directory = os.path.join(OONIPROBE_ROOT, 'ui', 'web','client')
+        self.web_ui_directory = os.path.join(OONIPROBE_ROOT, 'ui', 'web',
+                                             'client')
 
         self.inputs_directory = os.path.join(self.running_path, 'inputs')
         self.scheduler_directory = os.path.join(self.running_path, 'scheduler')
@@ -297,12 +364,13 @@ class OConfig(object):
         self.measurements_directory = os.path.join(self.running_path,
                                                    'measurements')
 
+        self.config_files = [
+            self.user_config_file_path,
+            '/etc/ooniprobe.conf'
+        ]
         if self.global_options.get('configfile'):
             config_file = self.global_options['configfile']
-            self.config_file = expanduser(config_file)
-        else:
-            self.config_file = os.path.join(self.running_path,
-                                            'ooniprobe.conf')
+            self.config_files.insert(0, expanduser(config_file))
 
         if 'logfile' in self.basic:
             self.basic.logfile = expanduser(
@@ -336,6 +404,14 @@ class OConfig(object):
                 if exc.errno != errno.EEXIST:
                     raise
 
+        # This means ooniprobe was installed for the first time or is coming
+        # from a 1.x series installation. We should configure the default deck.
+        if self.last_run_version.public == "0":
+            from ooni.deck.store import deck_store
+            DEFAULT_DECKS = ['web-full']
+            for deck_id in DEFAULT_DECKS:
+                deck_store.enable(deck_id)
+
     def create_config_file(self, include_ip=False, include_asn=True,
                            include_country=True, should_upload=True,
                            preferred_backend="onion"):
@@ -354,9 +430,10 @@ class OConfig(object):
         should_upload = _bool_to_yaml(should_upload)
 
         logfile = os.path.join(self.running_path, 'ooniprobe.log')
-        with open(self.config_file, 'w+') as out_file:
+        with open(self.user_config_file_path, 'w') as out_file:
             out_file.write(
-                    CONFIG_FILE_TEMPLATE.format(logfile=logfile,
+                    CONFIG_FILE_TEMPLATE.format(
+                                    logfile=logfile,
                                     include_ip=include_ip,
                                     include_asn=include_asn,
                                     include_country=include_country,
@@ -366,23 +443,12 @@ class OConfig(object):
         self.read_config_file()
 
     def read_config_file(self, check_incoherences=False):
-        configuration = {}
-        config_file = {}
-        log.debug("Reading config file from %s" % self.config_file)
-        if os.path.isfile(self.config_file):
-            with open(self.config_file) as f:
-                config_file_contents = '\n'.join(f.readlines())
-                config_file = yaml.safe_load(config_file_contents)
-
-        for category in defaults.keys():
-            configuration[category] = {}
-            for k, v in defaults[category].items():
-                try:
-                    value = config_file.get(category, {})[k]
-                except KeyError:
-                    value = v
-                configuration[category][k] = value
-                getattr(self, category)[k] = value
+        configuration = _load_config_files_with_defaults(
+            self.config_files, defaults)
+
+        for category in configuration.keys():
+            for key, value in configuration[category].items():
+                getattr(self, category)[key] = value
 
         self.set_paths()
         if check_incoherences:
@@ -405,7 +471,8 @@ class OConfig(object):
                 incoherent_pretty = ", ".join(incoherences[:-1]) + ' and ' + incoherences[-1]
             else:
                 incoherent_pretty = incoherences[0]
-            log.err("You must set properly %s in %s." % (incoherent_pretty, self.config_file))
+            log.err("You must set properly %s in %s." % (incoherent_pretty,
+                                                         self.config_files[0]))
             raise errors.ConfigFileIncoherent
 
     @defer.inlineCallbacks
@@ -439,7 +506,3 @@ class OConfig(object):
             self.log_incoherences(incoherent)
 
 config = OConfig()
-if not os.path.isfile(config.config_file) \
-       and os.path.isfile('/etc/ooniprobe.conf'):
-    config.global_options['configfile'] = '/etc/ooniprobe.conf'
-    config.set_paths()
diff --git a/ooni/tests/test_settings.py b/ooni/tests/test_settings.py
index f94ca14..30d9e19 100644
--- a/ooni/tests/test_settings.py
+++ b/ooni/tests/test_settings.py
@@ -1,4 +1,8 @@
+import os
 import random
+import tempfile
+
+import yaml
 
 from twisted.internet import defer, reactor
 from twisted.internet.protocol import Protocol, Factory
@@ -10,6 +14,7 @@ from ooni import errors
 from ooni.utils import net
 from bases import ConfigTestCase
 
+from ooni.settings import _load_config_files_with_defaults
 
 class TestSettings(ConfigTestCase):
     def setUp(self):
@@ -133,3 +138,59 @@ class TestSettings(ConfigTestCase):
 
         self.configuration['advanced']['interface'] = random.choice(get_if_list())
         self.conf.check_incoherences(self.configuration)
+
+
+    def test_load_config_files(self):
+        defaults = {
+            'cat1': {
+                'key': 'value'
+            },
+            'cat2': {
+                'key': 'value'
+            },
+            'cat3': {
+                'key': 'value'
+            }
+        }
+        config_file_A = {
+            'cat1': {
+                'key': 'valueA'
+            },
+            'cat2': {
+                'key': 'valueA',
+                'invalid_key': 'ignored'
+            },
+            'invalid_category': {
+                'ignored': 'ignored'
+            }
+        }
+        config_file_B = {
+            'cat1': {
+                'key': 'valueB'
+            }
+        }
+        temp_dir = tempfile.mkdtemp()
+        config_file_A_path = os.path.join(temp_dir, "configA.conf")
+        config_file_B_path = os.path.join(temp_dir, "configB.conf")
+        with open(config_file_A_path, 'w') as out_file:
+            yaml.safe_dump(config_file_A, out_file)
+
+        with open(config_file_B_path, 'w') as out_file:
+            yaml.safe_dump(config_file_B, out_file)
+
+        config = _load_config_files_with_defaults([config_file_B_path,
+                                                   '/invalid/path/ignored.txt',
+                                                   config_file_A_path],
+                                                  defaults)
+
+        self.assertEqual(config, {
+            'cat1': {
+                'key': 'valueB'
+            },
+            'cat2': {
+                'key': 'valueA'
+            },
+            'cat3': {
+                'key': 'value'
+            }
+        })
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 3ad9605..e7a0d88 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -330,7 +330,7 @@ def createDeck(global_options, url=None):
     except errors.MissingRequiredOption as option_name:
         log.err('Missing required option: "%s"' % option_name)
         incomplete_net_test_loader = option_name.net_test_loader
-        print incomplete_net_test_loader.usageOptions().getUsage()
+        map(log.msg, incomplete_net_test_loader.usageOptions().getUsage().split("\n"))
         raise SystemExit(2)
 
     except errors.NetTestNotFound as path:
@@ -508,7 +508,7 @@ def runWithDaemonDirector(global_options):
                                         url=data['url'].encode('utf8'))
                 # When the test has been completed, go back to waiting for a message.
                 d.addCallback(readmsg, channel, queue_object, consumer_tag, counter+1)
-            except exceptions.AMQPError,v:
+            except exceptions.AMQPError, v:
                 log.msg("Error")
                 log.exception(v)
                 finished.errback(v)
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index ad2dc50..ecb4cdb 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
     <app>
       Loading...
     </app>
-  <script type="text/javascript" src="app.bundle.js?f06164bd3b339e781c75"></script></body>
+  <script type="text/javascript" src="app.bundle.js?5432cc07ccbd2f5613be"></script></body>
 </html>
diff --git a/setup.py b/setup.py
index e5dd836..4c0bcfe 100644
--- a/setup.py
+++ b/setup.py
@@ -87,39 +87,54 @@ Have fun!
 from __future__ import print_function
 
 import os
-import shutil
 import tempfile
-import subprocess
 from glob import glob
 
 from ConfigParser import SafeConfigParser
 
 from os.path import join as pj
 from setuptools import setup
-from setuptools.command.install import install
+from setuptools.command.install import install as InstallCommand
 
 from ooni import __version__, __author__
 
-GEOIP_ASN_URL = "https://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz";
-GEOIP_URL = "https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz";
-TEST_LISTS_URL = "https://github.com/citizenlab/test-lists/archive/master.zip";
-
-def run_command(args, cwd=None):
-    try:
-        p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
-    except EnvironmentError:
-        return None
-    stdout = p.communicate()[0].strip()
-    if p.returncode != 0:
-        return None
-    return stdout
-
-def is_lepidopter():
-    if os.path.exists("/etc/default/lepidopter"):
-        return True
-    return False
-
-class OoniInstall(install):
+CLASSIFIERS = """\
+Development Status :: 5 - Production/Stable
+Environment :: Console
+Framework :: Twisted
+Intended Audience :: Developers
+Intended Audience :: Education
+Intended Audience :: End Users/Desktop
+Intended Audience :: Information Technology
+Intended Audience :: Science/Research
+Intended Audience :: Telecommunications Industry
+License :: OSI Approved :: BSD Licens
+Programming Language :: Python
+Programming Language :: Python :: 2
+Programming Language :: Python :: 2 :: Only
+Programming Language :: Python :: 2.6
+Programming Language :: Python :: 2.7
+Operating System :: MacOS :: MacOS X
+Operating System :: POSIX
+Operating System :: POSIX :: BSD
+Operating System :: POSIX :: BSD :: BSD/OS
+Operating System :: POSIX :: BSD :: FreeBSD
+Operating System :: POSIX :: BSD :: NetBSD
+Operating System :: POSIX :: BSD :: OpenBSD
+Operating System :: POSIX :: Linux
+Operating System :: Unix
+Topic :: Scientific/Engineering :: Information Analysis
+Topic :: Security
+Topic :: Security :: Cryptography
+Topic :: Software Development :: Libraries :: Application Frameworks
+Topic :: Software Development :: Libraries :: Python Modules
+Topic :: Software Development :: Testing
+Topic :: Software Development :: Testing :: Traffic Generation
+Topic :: System :: Networking :: Monitoring
+"""
+
+
+class OoniInstall(InstallCommand):
     def gen_config(self, share_path):
         config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample")
         o = open(config_file, "w+")
@@ -164,127 +179,90 @@ class OoniInstall(install):
         except OSError:
             pass
 
-    def ooniresources(self):
-        from ooni.resources import check_for_update
-        from twisted.internet import task
-        task.react(lambda _: check_for_update())
+    def pre_install(self):
+        prefix = os.path.abspath(self.prefix)
+        self.set_data_files(prefix)
 
-    def update_lepidopter_config(self):
-        try:
-            shutil.copyfile("data/configs/lepidopter-ooniprobe.conf",
-                            "/etc/ooniprobe/ooniprobe.conf")
-            shutil.copyfile("data/configs/lepidopter-oonireport.conf",
-                            "/etc/ooniprobe/oonireport.conf")
-        except Exception:
-            print("ERR: Failed to copy configuration files to /etc/ooniprobe/")
+    def post_install(self):
+        pass
 
     def run(self):
-        prefix = os.path.abspath(self.prefix)
-        self.set_data_files(prefix)
+        self.pre_install()
         self.do_egg_install()
-        self.ooniresources()
-        if is_lepidopter():
-            self.update_lepidopter_config()
-
-setup_requires = ['twisted', 'pyyaml']
-install_requires = []
-dependency_links = []
-data_files = []
-packages = [
-    'ooni',
-    'ooni.agent',
-    'ooni.common',
-    'ooni.contrib',
-    'ooni.contrib.dateutil',
-    'ooni.contrib.dateutil.tz',
-    'ooni.deck',
-    'ooni.kit',
-    'ooni.nettests',
-    'ooni.nettests.manipulation',
-    'ooni.nettests.experimental',
-    'ooni.nettests.scanning',
-    'ooni.nettests.blocking',
-    'ooni.nettests.third_party',
-    'ooni.scripts',
-    'ooni.templates',
-    'ooni.tests',
-    'ooni.ui',
-    'ooni.ui.web',
-    'ooni.utils'
-]
-
-with open('requirements.txt') as f:
-    for line in f:
-        if line.startswith("#"):
-            continue
-        if line.startswith('https'):
-            dependency_links.append(line)
-            continue
-        install_requires.append(line)
-
-setup(
-    name="ooniprobe",
-    version=__version__,
-    author=__author__,
-    author_email="contact@xxxxxxxxxxxxxxxxxxx",
-    description="Network measurement tool for"
-                "identifying traffic manipulation and blocking.",
-    long_description=__doc__,
-    license='BSD 2 clause',
-    url="https://ooni.torproject.org/";,
-    package_dir={'ooni': 'ooni'},
-    data_files=data_files,
-    packages=packages,
-    include_package_data=True,
-    dependency_links=dependency_links,
-    install_requires=install_requires,
-    setup_requires=setup_requires,
-    zip_safe=False,
-    entry_points={
-        'console_scripts': [
-            'ooniresources = ooni.scripts.ooniresources:run', # This is deprecated
-            'oonideckgen = ooni.scripts.oonideckgen:run', # This is deprecated
-
-            'ooniprobe = ooni.scripts.ooniprobe:run',
-            'oonireport = ooni.scripts.oonireport:run',
-            'ooniprobe-agent = ooni.scripts.ooniprobe_agent:run'
-        ]
-    },
-    cmdclass={
-        "install": OoniInstall
-    },
-    classifiers=(
-        "Development Status :: 5 - Production/Stable",
-        "Environment :: Console",
-        "Framework :: Twisted",
-        "Intended Audience :: Developers",
-        "Intended Audience :: Education",
-        "Intended Audience :: End Users/Desktop",
-        "Intended Audience :: Information Technology",
-        "Intended Audience :: Science/Research",
-        "Intended Audience :: Telecommunications Industry",
-        "License :: OSI Approved :: BSD License"
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2 :: Only",
-        "Programming Language :: Python :: 2.6",
-        "Programming Language :: Python :: 2.7",
-        "Operating System :: MacOS :: MacOS X",
-        "Operating System :: POSIX",
-        "Operating System :: POSIX :: BSD",
-        "Operating System :: POSIX :: BSD :: BSD/OS",
-        "Operating System :: POSIX :: BSD :: FreeBSD",
-        "Operating System :: POSIX :: BSD :: NetBSD",
-        "Operating System :: POSIX :: BSD :: OpenBSD",
-        "Operating System :: POSIX :: Linux",
-        "Operating System :: Unix",
-        "Topic :: Scientific/Engineering :: Information Analysis",
-        "Topic :: Security",
-        "Topic :: Security :: Cryptography",
-        "Topic :: Software Development :: Libraries :: Application Frameworks",
-        "Topic :: Software Development :: Libraries :: Python Modules",
-        "Topic :: Software Development :: Testing",
-        "Topic :: Software Development :: Testing :: Traffic Generation",
-        "Topic :: System :: Networking :: Monitoring",
+        self.post_install()
+
+def setup_package():
+    setup_requires = []
+    install_requires = []
+    dependency_links = []
+    data_files = []
+    packages = [
+        'ooni',
+        'ooni.agent',
+        'ooni.common',
+        'ooni.contrib',
+        'ooni.contrib.dateutil',
+        'ooni.contrib.dateutil.tz',
+        'ooni.deck',
+        'ooni.kit',
+        'ooni.nettests',
+        'ooni.nettests.manipulation',
+        'ooni.nettests.experimental',
+        'ooni.nettests.scanning',
+        'ooni.nettests.blocking',
+        'ooni.nettests.third_party',
+        'ooni.scripts',
+        'ooni.templates',
+        'ooni.tests',
+        'ooni.ui',
+        'ooni.ui.web',
+        'ooni.utils'
+    ]
+
+    with open('requirements.txt') as f:
+        for line in f:
+            if line.startswith("#"):
+                continue
+            if line.startswith('https'):
+                dependency_links.append(line)
+                continue
+            install_requires.append(line)
+
+    metadata = dict(
+        name="ooniprobe",
+        version=__version__,
+        author=__author__,
+        author_email="contact@xxxxxxxxxxxxxxxxxxx",
+        description="Network measurement tool for"
+                    "identifying traffic manipulation and blocking.",
+        long_description=__doc__,
+        license='BSD 2 clause',
+        url="https://ooni.torproject.org/";,
+        package_dir={'ooni': 'ooni'},
+        data_files=data_files,
+        packages=packages,
+        include_package_data=True,
+        dependency_links=dependency_links,
+        install_requires=install_requires,
+        setup_requires=setup_requires,
+        zip_safe=False,
+        entry_points={
+            'console_scripts': [
+                'ooniresources = ooni.scripts.ooniresources:run',  # This is deprecated
+                'oonideckgen = ooni.scripts.oonideckgen:run',  # This is deprecated
+
+                'ooniprobe = ooni.scripts.ooniprobe:run',
+                'oonireport = ooni.scripts.oonireport:run',
+                'ooniprobe-agent = ooni.scripts.ooniprobe_agent:run'
+            ]
+        },
+        cmdclass={
+            "install": OoniInstall
+        },
+        classifiers=[c for c in CLASSIFIERS.split('\n') if c]
     )
-)
+
+    setup(**metadata)
+
+if __name__ == "__main__":
+    setup_package()



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits