[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Add support for parsing multiple config files
commit 31bcae623b8a621aa941f284e67438168243c44a
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date: Fri Sep 2 15:00:48 2016 +0200
Add support for parsing multiple config files
* Configuration files are parsed in a certain order and they override the
default behavior.
---
Vagrantfile | 5 +
ooni/__init__.py | 2 +-
ooni/scripts/ooniprobe_agent.py | 1 +
ooni/settings.py | 125 ++++++++++++++-----
ooni/tests/test_settings.py | 61 ++++++++++
ooni/ui/cli.py | 4 +-
ooni/ui/web/client/index.html | 2 +-
setup.py | 262 ++++++++++++++++++----------------------
8 files changed, 285 insertions(+), 177 deletions(-)
diff --git a/Vagrantfile b/Vagrantfile
index 71f7c7a..8ff0bbd 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -81,4 +81,9 @@ Vagrant.configure("2") do |config|
end
end
+ config.vm.define "testing" do |testing|
+ testing.vm.network "forwarded_port", guest: 8842, host: 8142
+ testing.vm.synced_folder ".", "/data/ooni-probe"
+ end
+
end
diff --git a/ooni/__init__.py b/ooni/__init__.py
index 9eecb67..d4b6f95 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
# -*- encoding: utf-8 -*-
__author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0-alpha.2"
+__version__ = "2.0.0-alpha.3"
__all__ = [
'agent',
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index f56f2df..3eb1d22 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -58,6 +58,7 @@ def start_agent(options=None):
# a null log observer
twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull',
'--umask', '022']
+
twistd_config = OoniprobeTwistdConfig()
if options is not None:
twistd_args.extend(options.twistd_args)
diff --git a/ooni/settings.py b/ooni/settings.py
index a0ef0a5..66bd212 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -3,6 +3,7 @@ import sys
import yaml
import errno
import getpass
+from pkg_resources import parse_version
from ConfigParser import SafeConfigParser
from twisted.internet import defer, reactor
@@ -10,6 +11,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
from os.path import abspath, expanduser
+from ooni import __version__ as ooniprobe_version
from ooni.utils import Storage, log, get_ooni_root
CONFIG_FILE_TEMPLATE = """\
@@ -19,8 +21,8 @@ CONFIG_FILE_TEMPLATE = """\
basic:
# Where OONIProbe should be writing it's log file
- logfile: {logfile}
- loglevel: WARNING
+ # logfile: {logfile}
+ # loglevel: WARNING
# The maximum amount of data to store on disk. Once the quota is reached,
# we will start deleting older reports.
# measurement_quota: 1G
@@ -191,6 +193,37 @@ elif os.path.isfile(_SETTINGS_INI):
if _ETC_PATH is not None:
ETC_PATH = _ETC_PATH
+
+def _load_config_files_with_defaults(config_files, defaults):
+ """
+ This takes care of reading the config files in reverse order (the first
+ item will have priority over the last element) and produce a
+ configuration that includes ONLY the options inside of the defaults
+ dictionary.
+
+ :param config_files: a list of configuration file paths
+ :param defaults: the default values for the configuration file
+ :return: a configuration that is the result of reading the config files
+ and joining it with the default options.
+ """
+ config_from_files = {}
+ configuration = {}
+ for config_file_path in reversed(config_files):
+ if not os.path.exists(config_file_path):
+ continue
+ with open(config_file_path) as in_file:
+ c = yaml.safe_load(in_file)
+ config_from_files.update(c)
+
+ for category in defaults.keys():
+ configuration[category] = {}
+ for k, v in defaults[category].items():
+ try:
+ configuration[category][k] = config_from_files[category][k]
+ except KeyError:
+ configuration[category][k] = defaults[category][k]
+ return configuration
+
class OConfig(object):
_custom_home = None
@@ -211,6 +244,10 @@ class OConfig(object):
self.tor = Storage()
self.privacy = Storage()
+ # In here we store the configuration files ordered by priority.
+ # First configuration file takes priority over the others.
+ self.config_files = []
+
self.set_paths()
def is_initialized(self):
@@ -225,6 +262,31 @@ class OConfig(object):
with open(initialized_path, 'w+'): pass
@property
+ def last_run_version(self):
+ """
+ :return: Version identifying the last run version of ooniprobe.
+ """
+ last_run_version_path = os.path.join(
+ self.running_path, "last_run_version"
+ )
+ if not os.path.exists(last_run_version_path):
+ return parse_version("0")
+ with open(last_run_version_path) as in_file:
+ last_run_version = in_file.read()
+ return parse_version(last_run_version)
+
+ @property
+ def current_version(self):
+ return parse_version(ooniprobe_version)
+
+ def set_last_run_version(self):
+ last_run_version_path = os.path.join(
+ self.running_path, "last_run_version"
+ )
+ with open(last_run_version_path, "w") as out_file:
+ out_file.write(ooniprobe_version)
+
+ @property
def running_path(self):
"""
This is the directory used to store state application data.
@@ -266,6 +328,10 @@ class OConfig(object):
return VAR_LIB_PATH
@property
+ def user_config_file_path(self):
+ return os.path.join(self.running_path, 'ooniprobe.conf')
+
+ @property
def ooni_home(self):
home = expanduser('~'+self.current_user)
if os.getenv("HOME"):
@@ -283,7 +349,8 @@ class OConfig(object):
def set_paths(self):
self.nettest_directory = os.path.join(OONIPROBE_ROOT, 'nettests')
- self.web_ui_directory = os.path.join(OONIPROBE_ROOT, 'ui', 'web','client')
+ self.web_ui_directory = os.path.join(OONIPROBE_ROOT, 'ui', 'web',
+ 'client')
self.inputs_directory = os.path.join(self.running_path, 'inputs')
self.scheduler_directory = os.path.join(self.running_path, 'scheduler')
@@ -297,12 +364,13 @@ class OConfig(object):
self.measurements_directory = os.path.join(self.running_path,
'measurements')
+ self.config_files = [
+ self.user_config_file_path,
+ '/etc/ooniprobe.conf'
+ ]
if self.global_options.get('configfile'):
config_file = self.global_options['configfile']
- self.config_file = expanduser(config_file)
- else:
- self.config_file = os.path.join(self.running_path,
- 'ooniprobe.conf')
+ self.config_files.insert(0, expanduser(config_file))
if 'logfile' in self.basic:
self.basic.logfile = expanduser(
@@ -336,6 +404,14 @@ class OConfig(object):
if exc.errno != errno.EEXIST:
raise
+ # This means ooniprobe was installed for the first time or is coming
+ # from a 1.x series installation. We should configure the default deck.
+ if self.last_run_version.public == "0":
+ from ooni.deck.store import deck_store
+ DEFAULT_DECKS = ['web-full']
+ for deck_id in DEFAULT_DECKS:
+ deck_store.enable(deck_id)
+
def create_config_file(self, include_ip=False, include_asn=True,
include_country=True, should_upload=True,
preferred_backend="onion"):
@@ -354,9 +430,10 @@ class OConfig(object):
should_upload = _bool_to_yaml(should_upload)
logfile = os.path.join(self.running_path, 'ooniprobe.log')
- with open(self.config_file, 'w+') as out_file:
+ with open(self.user_config_file_path, 'w') as out_file:
out_file.write(
- CONFIG_FILE_TEMPLATE.format(logfile=logfile,
+ CONFIG_FILE_TEMPLATE.format(
+ logfile=logfile,
include_ip=include_ip,
include_asn=include_asn,
include_country=include_country,
@@ -366,23 +443,12 @@ class OConfig(object):
self.read_config_file()
def read_config_file(self, check_incoherences=False):
- configuration = {}
- config_file = {}
- log.debug("Reading config file from %s" % self.config_file)
- if os.path.isfile(self.config_file):
- with open(self.config_file) as f:
- config_file_contents = '\n'.join(f.readlines())
- config_file = yaml.safe_load(config_file_contents)
-
- for category in defaults.keys():
- configuration[category] = {}
- for k, v in defaults[category].items():
- try:
- value = config_file.get(category, {})[k]
- except KeyError:
- value = v
- configuration[category][k] = value
- getattr(self, category)[k] = value
+ configuration = _load_config_files_with_defaults(
+ self.config_files, defaults)
+
+ for category in configuration.keys():
+ for key, value in configuration[category].items():
+ getattr(self, category)[key] = value
self.set_paths()
if check_incoherences:
@@ -405,7 +471,8 @@ class OConfig(object):
incoherent_pretty = ", ".join(incoherences[:-1]) + ' and ' + incoherences[-1]
else:
incoherent_pretty = incoherences[0]
- log.err("You must set properly %s in %s." % (incoherent_pretty, self.config_file))
+ log.err("You must set properly %s in %s." % (incoherent_pretty,
+ self.config_files[0]))
raise errors.ConfigFileIncoherent
@defer.inlineCallbacks
@@ -439,7 +506,3 @@ class OConfig(object):
self.log_incoherences(incoherent)
config = OConfig()
-if not os.path.isfile(config.config_file) \
- and os.path.isfile('/etc/ooniprobe.conf'):
- config.global_options['configfile'] = '/etc/ooniprobe.conf'
- config.set_paths()
diff --git a/ooni/tests/test_settings.py b/ooni/tests/test_settings.py
index f94ca14..30d9e19 100644
--- a/ooni/tests/test_settings.py
+++ b/ooni/tests/test_settings.py
@@ -1,4 +1,8 @@
+import os
import random
+import tempfile
+
+import yaml
from twisted.internet import defer, reactor
from twisted.internet.protocol import Protocol, Factory
@@ -10,6 +14,7 @@ from ooni import errors
from ooni.utils import net
from bases import ConfigTestCase
+from ooni.settings import _load_config_files_with_defaults
class TestSettings(ConfigTestCase):
def setUp(self):
@@ -133,3 +138,59 @@ class TestSettings(ConfigTestCase):
self.configuration['advanced']['interface'] = random.choice(get_if_list())
self.conf.check_incoherences(self.configuration)
+
+
+ def test_load_config_files(self):
+ defaults = {
+ 'cat1': {
+ 'key': 'value'
+ },
+ 'cat2': {
+ 'key': 'value'
+ },
+ 'cat3': {
+ 'key': 'value'
+ }
+ }
+ config_file_A = {
+ 'cat1': {
+ 'key': 'valueA'
+ },
+ 'cat2': {
+ 'key': 'valueA',
+ 'invalid_key': 'ignored'
+ },
+ 'invalid_category': {
+ 'ignored': 'ignored'
+ }
+ }
+ config_file_B = {
+ 'cat1': {
+ 'key': 'valueB'
+ }
+ }
+ temp_dir = tempfile.mkdtemp()
+ config_file_A_path = os.path.join(temp_dir, "configA.conf")
+ config_file_B_path = os.path.join(temp_dir, "configB.conf")
+ with open(config_file_A_path, 'w') as out_file:
+ yaml.safe_dump(config_file_A, out_file)
+
+ with open(config_file_B_path, 'w') as out_file:
+ yaml.safe_dump(config_file_B, out_file)
+
+ config = _load_config_files_with_defaults([config_file_B_path,
+ '/invalid/path/ignored.txt',
+ config_file_A_path],
+ defaults)
+
+ self.assertEqual(config, {
+ 'cat1': {
+ 'key': 'valueB'
+ },
+ 'cat2': {
+ 'key': 'valueA'
+ },
+ 'cat3': {
+ 'key': 'value'
+ }
+ })
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 3ad9605..e7a0d88 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -330,7 +330,7 @@ def createDeck(global_options, url=None):
except errors.MissingRequiredOption as option_name:
log.err('Missing required option: "%s"' % option_name)
incomplete_net_test_loader = option_name.net_test_loader
- print incomplete_net_test_loader.usageOptions().getUsage()
+ map(log.msg, incomplete_net_test_loader.usageOptions().getUsage().split("\n"))
raise SystemExit(2)
except errors.NetTestNotFound as path:
@@ -508,7 +508,7 @@ def runWithDaemonDirector(global_options):
url=data['url'].encode('utf8'))
# When the test has been completed, go back to waiting for a message.
d.addCallback(readmsg, channel, queue_object, consumer_tag, counter+1)
- except exceptions.AMQPError,v:
+ except exceptions.AMQPError, v:
log.msg("Error")
log.exception(v)
finished.errback(v)
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index ad2dc50..ecb4cdb 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
<app>
Loading...
</app>
- <script type="text/javascript" src="app.bundle.js?f06164bd3b339e781c75"></script></body>
+ <script type="text/javascript" src="app.bundle.js?5432cc07ccbd2f5613be"></script></body>
</html>
diff --git a/setup.py b/setup.py
index e5dd836..4c0bcfe 100644
--- a/setup.py
+++ b/setup.py
@@ -87,39 +87,54 @@ Have fun!
from __future__ import print_function
import os
-import shutil
import tempfile
-import subprocess
from glob import glob
from ConfigParser import SafeConfigParser
from os.path import join as pj
from setuptools import setup
-from setuptools.command.install import install
+from setuptools.command.install import install as InstallCommand
from ooni import __version__, __author__
-GEOIP_ASN_URL = "https://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz";
-GEOIP_URL = "https://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz";
-TEST_LISTS_URL = "https://github.com/citizenlab/test-lists/archive/master.zip";
-
-def run_command(args, cwd=None):
- try:
- p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
- except EnvironmentError:
- return None
- stdout = p.communicate()[0].strip()
- if p.returncode != 0:
- return None
- return stdout
-
-def is_lepidopter():
- if os.path.exists("/etc/default/lepidopter"):
- return True
- return False
-
-class OoniInstall(install):
+CLASSIFIERS = """\
+Development Status :: 5 - Production/Stable
+Environment :: Console
+Framework :: Twisted
+Intended Audience :: Developers
+Intended Audience :: Education
+Intended Audience :: End Users/Desktop
+Intended Audience :: Information Technology
+Intended Audience :: Science/Research
+Intended Audience :: Telecommunications Industry
+License :: OSI Approved :: BSD Licens
+Programming Language :: Python
+Programming Language :: Python :: 2
+Programming Language :: Python :: 2 :: Only
+Programming Language :: Python :: 2.6
+Programming Language :: Python :: 2.7
+Operating System :: MacOS :: MacOS X
+Operating System :: POSIX
+Operating System :: POSIX :: BSD
+Operating System :: POSIX :: BSD :: BSD/OS
+Operating System :: POSIX :: BSD :: FreeBSD
+Operating System :: POSIX :: BSD :: NetBSD
+Operating System :: POSIX :: BSD :: OpenBSD
+Operating System :: POSIX :: Linux
+Operating System :: Unix
+Topic :: Scientific/Engineering :: Information Analysis
+Topic :: Security
+Topic :: Security :: Cryptography
+Topic :: Software Development :: Libraries :: Application Frameworks
+Topic :: Software Development :: Libraries :: Python Modules
+Topic :: Software Development :: Testing
+Topic :: Software Development :: Testing :: Traffic Generation
+Topic :: System :: Networking :: Monitoring
+"""
+
+
+class OoniInstall(InstallCommand):
def gen_config(self, share_path):
config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample")
o = open(config_file, "w+")
@@ -164,127 +179,90 @@ class OoniInstall(install):
except OSError:
pass
- def ooniresources(self):
- from ooni.resources import check_for_update
- from twisted.internet import task
- task.react(lambda _: check_for_update())
+ def pre_install(self):
+ prefix = os.path.abspath(self.prefix)
+ self.set_data_files(prefix)
- def update_lepidopter_config(self):
- try:
- shutil.copyfile("data/configs/lepidopter-ooniprobe.conf",
- "/etc/ooniprobe/ooniprobe.conf")
- shutil.copyfile("data/configs/lepidopter-oonireport.conf",
- "/etc/ooniprobe/oonireport.conf")
- except Exception:
- print("ERR: Failed to copy configuration files to /etc/ooniprobe/")
+ def post_install(self):
+ pass
def run(self):
- prefix = os.path.abspath(self.prefix)
- self.set_data_files(prefix)
+ self.pre_install()
self.do_egg_install()
- self.ooniresources()
- if is_lepidopter():
- self.update_lepidopter_config()
-
-setup_requires = ['twisted', 'pyyaml']
-install_requires = []
-dependency_links = []
-data_files = []
-packages = [
- 'ooni',
- 'ooni.agent',
- 'ooni.common',
- 'ooni.contrib',
- 'ooni.contrib.dateutil',
- 'ooni.contrib.dateutil.tz',
- 'ooni.deck',
- 'ooni.kit',
- 'ooni.nettests',
- 'ooni.nettests.manipulation',
- 'ooni.nettests.experimental',
- 'ooni.nettests.scanning',
- 'ooni.nettests.blocking',
- 'ooni.nettests.third_party',
- 'ooni.scripts',
- 'ooni.templates',
- 'ooni.tests',
- 'ooni.ui',
- 'ooni.ui.web',
- 'ooni.utils'
-]
-
-with open('requirements.txt') as f:
- for line in f:
- if line.startswith("#"):
- continue
- if line.startswith('https'):
- dependency_links.append(line)
- continue
- install_requires.append(line)
-
-setup(
- name="ooniprobe",
- version=__version__,
- author=__author__,
- author_email="contact@xxxxxxxxxxxxxxxxxxx",
- description="Network measurement tool for"
- "identifying traffic manipulation and blocking.",
- long_description=__doc__,
- license='BSD 2 clause',
- url="https://ooni.torproject.org/";,
- package_dir={'ooni': 'ooni'},
- data_files=data_files,
- packages=packages,
- include_package_data=True,
- dependency_links=dependency_links,
- install_requires=install_requires,
- setup_requires=setup_requires,
- zip_safe=False,
- entry_points={
- 'console_scripts': [
- 'ooniresources = ooni.scripts.ooniresources:run', # This is deprecated
- 'oonideckgen = ooni.scripts.oonideckgen:run', # This is deprecated
-
- 'ooniprobe = ooni.scripts.ooniprobe:run',
- 'oonireport = ooni.scripts.oonireport:run',
- 'ooniprobe-agent = ooni.scripts.ooniprobe_agent:run'
- ]
- },
- cmdclass={
- "install": OoniInstall
- },
- classifiers=(
- "Development Status :: 5 - Production/Stable",
- "Environment :: Console",
- "Framework :: Twisted",
- "Intended Audience :: Developers",
- "Intended Audience :: Education",
- "Intended Audience :: End Users/Desktop",
- "Intended Audience :: Information Technology",
- "Intended Audience :: Science/Research",
- "Intended Audience :: Telecommunications Industry",
- "License :: OSI Approved :: BSD License"
- "Programming Language :: Python",
- "Programming Language :: Python :: 2",
- "Programming Language :: Python :: 2 :: Only",
- "Programming Language :: Python :: 2.6",
- "Programming Language :: Python :: 2.7",
- "Operating System :: MacOS :: MacOS X",
- "Operating System :: POSIX",
- "Operating System :: POSIX :: BSD",
- "Operating System :: POSIX :: BSD :: BSD/OS",
- "Operating System :: POSIX :: BSD :: FreeBSD",
- "Operating System :: POSIX :: BSD :: NetBSD",
- "Operating System :: POSIX :: BSD :: OpenBSD",
- "Operating System :: POSIX :: Linux",
- "Operating System :: Unix",
- "Topic :: Scientific/Engineering :: Information Analysis",
- "Topic :: Security",
- "Topic :: Security :: Cryptography",
- "Topic :: Software Development :: Libraries :: Application Frameworks",
- "Topic :: Software Development :: Libraries :: Python Modules",
- "Topic :: Software Development :: Testing",
- "Topic :: Software Development :: Testing :: Traffic Generation",
- "Topic :: System :: Networking :: Monitoring",
+ self.post_install()
+
+def setup_package():
+ setup_requires = []
+ install_requires = []
+ dependency_links = []
+ data_files = []
+ packages = [
+ 'ooni',
+ 'ooni.agent',
+ 'ooni.common',
+ 'ooni.contrib',
+ 'ooni.contrib.dateutil',
+ 'ooni.contrib.dateutil.tz',
+ 'ooni.deck',
+ 'ooni.kit',
+ 'ooni.nettests',
+ 'ooni.nettests.manipulation',
+ 'ooni.nettests.experimental',
+ 'ooni.nettests.scanning',
+ 'ooni.nettests.blocking',
+ 'ooni.nettests.third_party',
+ 'ooni.scripts',
+ 'ooni.templates',
+ 'ooni.tests',
+ 'ooni.ui',
+ 'ooni.ui.web',
+ 'ooni.utils'
+ ]
+
+ with open('requirements.txt') as f:
+ for line in f:
+ if line.startswith("#"):
+ continue
+ if line.startswith('https'):
+ dependency_links.append(line)
+ continue
+ install_requires.append(line)
+
+ metadata = dict(
+ name="ooniprobe",
+ version=__version__,
+ author=__author__,
+ author_email="contact@xxxxxxxxxxxxxxxxxxx",
+ description="Network measurement tool for"
+ "identifying traffic manipulation and blocking.",
+ long_description=__doc__,
+ license='BSD 2 clause',
+ url="https://ooni.torproject.org/";,
+ package_dir={'ooni': 'ooni'},
+ data_files=data_files,
+ packages=packages,
+ include_package_data=True,
+ dependency_links=dependency_links,
+ install_requires=install_requires,
+ setup_requires=setup_requires,
+ zip_safe=False,
+ entry_points={
+ 'console_scripts': [
+ 'ooniresources = ooni.scripts.ooniresources:run', # This is deprecated
+ 'oonideckgen = ooni.scripts.oonideckgen:run', # This is deprecated
+
+ 'ooniprobe = ooni.scripts.ooniprobe:run',
+ 'oonireport = ooni.scripts.oonireport:run',
+ 'ooniprobe-agent = ooni.scripts.ooniprobe_agent:run'
+ ]
+ },
+ cmdclass={
+ "install": OoniInstall
+ },
+ classifiers=[c for c in CLASSIFIERS.split('\n') if c]
)
-)
+
+ setup(**metadata)
+
+if __name__ == "__main__":
+ setup_package()
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits