[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Implement deck store



commit 97ed0fcbda310db18b2138f70024f08de5b523b9
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Fri Jul 29 22:50:45 2016 +0200

    Implement deck store
    
    * Write all runtime files to /var/lib/ooni
    
    * Other various fixes to runtime paths
    
    * Include the default deck and data files in distribution
---
 MANIFEST.in                     |  2 ++
 data/decks/web.yaml             | 22 +++++++++++++
 ooni/agent/agent.py             |  8 -----
 ooni/agent/scheduler.py         | 22 ++++++++++++-
 ooni/deck/deck.py               | 15 +++++++--
 ooni/deck/store.py              | 35 +++++++++++++++++---
 ooni/scripts/ooniprobe_agent.py | 17 +++++++---
 ooni/settings.py                | 72 ++++++++++++++++++++++++++++-------------
 ooni/ui/web/server.py           |  8 +----
 ooni/utils/log.py               |  2 ++
 10 files changed, 153 insertions(+), 50 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 692c73f..485e834 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,4 +1,6 @@
+recursive-include ooni/ui/web/client *
 include README.rst ChangeLog.rst requirements.txt LICENSE
+recursive-include data/decks *
 include data/oonideckgen.1
 include data/ooniprobe.1
 include data/oonireport.1
diff --git a/data/decks/web.yaml b/data/decks/web.yaml
new file mode 100644
index 0000000..a81b8f8
--- /dev/null
+++ b/data/decks/web.yaml
@@ -0,0 +1,22 @@
+---
+name: Web related ooniprobe tests
+description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
+    Request and the Web Connectivity test
+tasks:
+- name: Runs the HTTP Header Field Manipulation test
+  ooni:
+    test_name: http_header_field_manipulation
+
+- name: Runs the HTTP Invalid Request Line test
+  ooni:
+    test_name: http_invalid_request_line
+
+- name: Runs the Web Connectivity Test
+  ooni:
+    test_name: web_connectivity
+    file: $citizenlab_global_urls
+
+- name: Runs the Web Connectivity Test
+  ooni:
+    test_name: web_connectivity
+    file: $citizenlab_${probe_cc}_urls
diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index ff9a2dd..c2e7e26 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -1,7 +1,6 @@
 from twisted.application import service
 from ooni.director import Director
 from ooni.settings import config
-from ooni.utils import log
 
 from ooni.ui.web.web import WebUIService
 from ooni.agent.scheduler import SchedulerService
@@ -11,9 +10,6 @@ class AgentService(service.MultiService):
         service.MultiService.__init__(self)
 
         director = Director()
-        config.set_paths()
-        config.initialize_ooni_home()
-        config.read_config_file()
 
         self.web_ui_service = WebUIService(director, web_ui_port)
         self.web_ui_service.setServiceParent(self)
@@ -24,9 +20,5 @@ class AgentService(service.MultiService):
     def startService(self):
         service.MultiService.startService(self)
 
-        log.start()
-
     def stopService(self):
         service.MultiService.stopService(self)
-
-        log.stop()
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 167a14a..43715e8 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -7,7 +7,7 @@ from twisted.python.filepath import FilePath
 from ooni.scripts import oonireport
 from ooni import resources
 from ooni.utils import log, SHORT_DATE
-from ooni.deck.store import input_store
+from ooni.deck.store import input_store, deck_store
 from ooni.settings import config
 from ooni.contrib import croniter
 from ooni.geoip import probe_ip
@@ -116,6 +116,25 @@ class DeleteOldReports(ScheduledTask):
                 log.debug("Deleting old report {0}".format(measurement["id"]))
                 measurement_path.child(measurement['id']).remove()
 
+
+class RunDecks(ScheduledTask):
+    """
+    This will run the decks that have been configured on the system as the
+    decks to run by default.
+    """
+    schedule = '@daily'
+    identifier = 'run-decks'
+
+    def __init__(self, director, schedule=None):
+        super(RunDecks, self).__init__(schedule)
+        self.director = director
+
+    @defer.inlineCallbacks
+    def task(self):
+        for deck_id, deck in deck_store.list():
+            yield deck.setup()
+            yield deck.run(self.director)
+
 class SendHeartBeat(ScheduledTask):
     """
     This task is used to send a heartbeat that the probe is still alive and
@@ -193,6 +212,7 @@ class SchedulerService(service.MultiService):
         self.schedule(UpdateInputsAndResources())
         self.schedule(UploadReports())
         self.schedule(DeleteOldReports())
+        self.schedule(RunDecks(self.director))
 
         self._looping_call.start(self.interval)
 
diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index 868bfb8..1b2300a 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -1,5 +1,6 @@
 import os
 from copy import deepcopy
+from string import Template
 
 import yaml
 from twisted.internet import defer
@@ -10,7 +11,6 @@ from ooni.backend_client import BouncerClient, CollectorClient
 from ooni.backend_client import get_preferred_bouncer
 from ooni.deck.backend import lookup_collector_and_test_helpers
 from ooni.deck.legacy import convert_legacy_deck
-from ooni.deck.store import input_store
 from ooni.geoip import probe_ip
 from ooni.nettest import NetTestLoader, nettest_to_path
 from ooni.measurements import generate_summary
@@ -19,6 +19,7 @@ from ooni.utils import log, generate_filename
 
 
 def resolve_file_path(v, prepath=None):
+    from ooni.deck.store import input_store
     if v.startswith("$"):
         # This raises InputNotFound and we let it carry onto the caller
         return input_store.get(v[1:])["filepath"]
@@ -248,8 +249,13 @@ class NGDeck(object):
         """
         This method needs to be called before you are able to run a deck.
         """
+        from ooni.deck.store import InputNotFound
         for task in self._tasks:
-            yield task.setup()
+            try:
+                yield task.setup()
+            except InputNotFound:
+                log.msg("Skipping this task because the input cannot be found")
+                self._skip = True
         self._is_setup = True
 
     @defer.inlineCallbacks
@@ -365,7 +371,10 @@ class DeckTask(object):
     def _setup_ooni(self):
         yield probe_ip.lookup()
         for input_file in self.ooni['net_test_loader'].inputFiles:
-            file_path = resolve_file_path(input_file['filename'], self.cwd)
+            filename = Template(input_file['filename']).safe_substitute(
+                probe_cc=probe_ip.geodata['countrycode'].lower()
+            )
+            file_path = resolve_file_path(filename, self.cwd)
             input_file['test_options'][input_file['key']] = file_path
         self.ooni['test_details'] = self.ooni['net_test_loader'].getTestDetails()
         self.id = generate_filename(self.ooni['test_details'])
diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index 05c0b95..7c90204 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -5,13 +5,18 @@ from copy import deepcopy
 from twisted.internet import defer
 from twisted.python.filepath import FilePath
 
+from ooni.deck.deck import NGDeck
 from ooni.otime import timestampNowISO8601UTC
 from ooni.resources import check_for_update
 from ooni.settings import config
+from ooni.utils import log
 
 class InputNotFound(Exception):
     pass
 
+class DeckNotFound(Exception):
+    pass
+
 class InputStore(object):
     def __init__(self):
         self.path = FilePath(config.inputs_directory)
@@ -117,12 +122,34 @@ class InputStore(object):
 class DeckStore(object):
     def __init__(self):
         self.path = FilePath(config.decks_directory)
+        self._cache = {}
+        self._cache_stale = True
 
-    def update(self):
-        pass
+    def list(self):
+        decks = []
+        if self._cache_stale:
+            self._update_cache()
+        for deck_id, deck in self._cache.iteritems():
+            decks.append((deck_id, deck))
+        return decks
 
-    def get(self):
-        pass
+    def _update_cache(self):
+        for deck_path in self.path.listdir():
+            if not deck_path.endswith('.yaml'):
+                continue
+            deck_id = deck_path[:-1*len('.yaml')]
+            deck = NGDeck(
+                deck_path=self.path.child(deck_path).path
+            )
+            self._cache[deck_id] = deck
 
+    def get(self, deck_id):
+        if self._cache_stale:
+            self._update_cache()
+        try:
+            return self._cache[deck_id]
+        except KeyError:
+            raise DeckNotFound(deck_id)
 
+deck_store = DeckStore()
 input_store = InputStore()
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 3f8efc8..30df9b9 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -7,6 +7,7 @@ import signal
 from twisted.scripts import twistd
 from twisted.python import usage
 
+from ooni.utils import log
 from ooni.settings import config
 from ooni.agent.agent import AgentService
 
@@ -48,8 +49,15 @@ class AgentOptions(usage.Options):
         self.twistd_args = []
 
 def start_agent(options=None):
-    os.chdir(config.ooni_home)
-    twistd_args = []
+    config.set_paths()
+    config.initialize_ooni_home()
+    config.read_config_file()
+
+    os.chdir(config.running_path)
+
+    # Since we are starting the logger below ourselves we make twistd log to
+    #  a null log observer
+    twistd_args = ['--logger', 'ooni.utils.log.ooniloggerNull']
     twistd_config = OoniprobeTwistdConfig()
     if options is not None:
         twistd_args.extend(options.twistd_args)
@@ -63,12 +71,13 @@ def start_agent(options=None):
     }
     print("Starting ooniprobe agent.")
     print("To view the GUI go to %s" % WEB_UI_URL)
+    log.start()
     twistd.runApp(twistd_config)
     return 0
 
 def status_agent():
     pidfile = os.path.join(
-        config.ooni_home,
+        config.running_path,
         'twistd.pid'
     )
     if not os.path.exists(pidfile):
@@ -88,7 +97,7 @@ def status_agent():
 def stop_agent():
     # This function is borrowed from tahoe
     pidfile = os.path.join(
-        config.ooni_home,
+        config.running_path,
         'twistd.pid'
     )
     if not os.path.exists(pidfile):
diff --git a/ooni/settings.py b/ooni/settings.py
index 194729a..5762919 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -1,4 +1,5 @@
 import os
+import sys
 import yaml
 import getpass
 from ConfigParser import SafeConfigParser
@@ -12,7 +13,6 @@ from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol
 from ooni.utils import Storage, log, get_ooni_root
 from ooni import errors
 
-
 class OConfig(object):
     _custom_home = None
 
@@ -41,18 +41,56 @@ class OConfig(object):
 
     @property
     def var_lib_path(self):
+        if hasattr(sys, 'real_prefix'):
+            # We are in a virtualenv use the /usr/share in the virtualenv
+            return os.path.join(
+                os.path.abspath(sys.prefix),
+                'var', 'lib', 'ooni'
+            )
         var_lib_path = self.embedded_settings("directories", "var_lib")
         if var_lib_path:
             return os.path.abspath(var_lib_path)
         return "/var/lib/ooni"
 
     @property
+    def running_path(self):
+        """
+        This is the directory used to store state application data.
+        It defaults to /var/lib/ooni, but if that is not writeable we will
+        use the ooni_home.
+        """
+        var_lib_path = self.var_lib_path
+        if os.access(var_lib_path, os.W_OK):
+            return var_lib_path
+        return self.ooni_home
+
+    @property
     def usr_share_path(self):
+        if hasattr(sys, 'real_prefix'):
+            # We are in a virtualenv use the /usr/share in the virtualenv
+            return os.path.join(
+                os.path.abspath(sys.prefix),
+                'usr', 'share', 'ooni'
+            )
         usr_share_path = self.embedded_settings("directories", "usr_share")
         if usr_share_path:
             return os.path.abspath(usr_share_path)
         return "/usr/share/ooni"
 
+
+    @property
+    def etc_path(self):
+        if hasattr(sys, 'real_prefix'):
+            # We are in a virtualenv use the /usr/share in the virtualenv
+            return os.path.join(
+                os.path.abspath(sys.prefix),
+                'usr', 'share', 'ooni'
+            )
+        etc_path = self.embedded_settings("directories", "etc")
+        if etc_path:
+            return os.path.abspath(etc_path)
+        return "/etc"
+
     @property
     def data_directory_candidates(self):
         dirs = [
@@ -93,28 +131,15 @@ class OConfig(object):
 
     def set_paths(self):
         self.nettest_directory = os.path.join(get_ooni_root(), 'nettests')
+        self.web_ui_directory = os.path.join(get_ooni_root(), 'web', 'client')
 
-        if self.advanced.inputs_dir:
-            self.inputs_directory = self.advanced.inputs_dir
-        else:
-            self.inputs_directory = os.path.join(self.ooni_home, 'inputs')
-
-        self.scheduler_directory = os.path.join(self.ooni_home, 'scheduler')
+        self.inputs_directory = os.path.join(self.running_path, 'inputs')
+        self.scheduler_directory = os.path.join(self.running_path, 'scheduler')
+        self.decks_directory = os.path.join(self.running_path, 'decks')
+        self.resources_directory = os.path.join(self.running_path, 'resources')
 
-        if self.advanced.decks_dir:
-            self.decks_directory = self.advanced.decks_dir
-        else:
-            self.decks_directory = os.path.join(self.ooni_home, 'decks')
-
-        self.measurements_directory = os.path.join(self.ooni_home,
+        self.measurements_directory = os.path.join(self.running_path,
                                                    'measurements')
-        self.resources_directory = os.path.join(self.ooni_home,
-                                                "resources")
-        if self.advanced.report_log_file:
-            self.report_log_file = self.advanced.report_log_file
-        else:
-            self.report_log_file = os.path.join(self.ooni_home,
-                                                'reporting.yml')
 
         if self.global_options.get('configfile'):
             config_file = self.global_options['configfile']
@@ -123,8 +148,9 @@ class OConfig(object):
             self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf')
 
         if 'logfile' in self.basic:
-            self.basic.logfile = expanduser(self.basic.logfile.replace(
-                '~', '~'+self.current_user))
+            self.basic.logfile = expanduser(
+                self.basic.logfile.replace('~', '~'+self.current_user)
+            )
 
     def initialize_ooni_home(self, custom_home=None):
         if custom_home:
@@ -147,7 +173,7 @@ class OConfig(object):
         ]
         for path in sub_directories:
             try:
-                os.mkdir(path)
+                os.makedirs(path)
             except OSError as exc:
                 if exc.errno != 17:
                     raise
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index e1a6398..74cb235 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -26,11 +26,6 @@ from ooni.geoip import probe_ip
 
 config.advanced.debug = True
 
-def rpath(*path):
-    context = os.path.abspath(os.path.dirname(__file__))
-    return os.path.join(context, *path)
-
-
 class WebUIError(Exception):
     def __init__(self, code, message):
         self.code = code
@@ -392,5 +387,4 @@ class WebUIAPI(object):
     @app.route('/client/', branch=True)
     @xsrf_protect(check=False)
     def static(self, request):
-        path = rpath("client")
-        return static.File(path)
+        return static.File(config.web_ui_directory)
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 982a353..1aba8dd 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -158,6 +158,8 @@ class OONILogger(object):
         self.fileObserver.stop()
 
 oonilogger = OONILogger()
+# This is a mock of a LoggerObserverFactory to be supplied to twistd.
+ooniloggerNull = lambda: lambda eventDict: None
 
 start = oonilogger.start
 stop = oonilogger.stop



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits