[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Add support for deck lifecycle in the web UI



commit 7663106d25d272ffc7ff2208d95cddef3b685f62
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Thu Aug 4 19:46:26 2016 +0200

    Add support for deck lifecycle in the web UI
    
    * Fix a series of bugs found while testing
    * Bump the version number up one
---
 MANIFEST.in                    |   1 +
 data/decks/web-full.yaml       |  23 ++++++
 data/decks/web-no-invalid.yaml |  18 +++++
 data/decks/web.yaml            |  23 ------
 ooni/__init__.py               |   2 +-
 ooni/agent/agent.py            |   9 ++-
 ooni/agent/scheduler.py        |  64 ++++++++++++++---
 ooni/deck/deck.py              |   9 ++-
 ooni/deck/store.py             |   4 +-
 ooni/resources.py              |   5 ++
 ooni/scripts/oonideckgen.py    |  20 ++----
 ooni/scripts/ooniprobe.py      |   3 +-
 ooni/scripts/oonireport.py     |   7 +-
 ooni/settings.py               | 154 +++++++++++++++++++----------------------
 ooni/ui/cli.py                 |  37 ++++++++--
 ooni/ui/web/client/index.html  |   2 +-
 ooni/ui/web/server.py          |  28 +++++++-
 ooni/ui/web/web.py             |   5 +-
 ooni/utils/__init__.py         |   4 +-
 setup.py                       |  29 +++-----
 20 files changed, 271 insertions(+), 176 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 485e834..60d2ef9 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -9,3 +9,4 @@ include data/ooniprobe.conf.sample
 include data/configs/lepidopter-ooniprobe.conf
 include data/configs/lepidopter-oonireport.conf
 include ooni/settings.ini
+include ooni/ui/consent-form.md
diff --git a/data/decks/web-full.yaml b/data/decks/web-full.yaml
new file mode 100644
index 0000000..7812505
--- /dev/null
+++ b/data/decks/web-full.yaml
@@ -0,0 +1,23 @@
+---
+name: Full Web test deck
+description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
+    Request and the Web Connectivity test
+schedule: "@daily"
+tasks:
+- name: Runs the HTTP Header Field Manipulation test
+  ooni:
+    test_name: http_header_field_manipulation
+
+- name: Runs the HTTP Invalid Request Line test
+  ooni:
+    test_name: http_invalid_request_line
+
+- name: Runs the Web Connectivity Test
+  ooni:
+    test_name: web_connectivity
+    file: $citizenlab_global_urls
+
+- name: Runs the Web Connectivity Test
+  ooni:
+    test_name: web_connectivity
+    file: $citizenlab_${probe_cc}_urls
diff --git a/data/decks/web-no-invalid.yaml b/data/decks/web-no-invalid.yaml
new file mode 100644
index 0000000..ea93488
--- /dev/null
+++ b/data/decks/web-no-invalid.yaml
@@ -0,0 +1,18 @@
+---
+name: Web test deck without HTTP Invalid Request Line
+description: This deck runs HTTP Header Field Manipulation, and the Web Connectivity test
+schedule: "@daily"
+tasks:
+- name: Runs the HTTP Header Field Manipulation test
+  ooni:
+    test_name: http_header_field_manipulation
+
+- name: Runs the Web Connectivity Test
+  ooni:
+    test_name: web_connectivity
+    file: $citizenlab_global_urls
+
+- name: Runs the Web Connectivity Test
+  ooni:
+    test_name: web_connectivity
+    file: $citizenlab_${probe_cc}_urls
diff --git a/data/decks/web.yaml b/data/decks/web.yaml
deleted file mode 100644
index c7b9bdc..0000000
--- a/data/decks/web.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
----
-name: Web related ooniprobe tests
-description: This deck runs HTTP Header Field Manipulation, HTTP Invalid
-    Request and the Web Connectivity test
-schedule: "@daily"
-tasks:
-- name: Runs the HTTP Header Field Manipulation test
-  ooni:
-    test_name: http_header_field_manipulation
-
-- name: Runs the HTTP Invalid Request Line test
-  ooni:
-    test_name: http_invalid_request_line
-
-- name: Runs the Web Connectivity Test
-  ooni:
-    test_name: web_connectivity
-    file: $citizenlab_global_urls
-
-- name: Runs the Web Connectivity Test
-  ooni:
-    test_name: web_connectivity
-    file: $citizenlab_${probe_cc}_urls
diff --git a/ooni/__init__.py b/ooni/__init__.py
index 653a636..1a31608 100644
--- a/ooni/__init__.py
+++ b/ooni/__init__.py
@@ -1,7 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 __author__ = "Open Observatory of Network Interference"
-__version__ = "2.0.0a0"
+__version__ = "2.0.0a1"
 
 __all__ = [
     'agent',
diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index c2e7e26..0311cef 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -11,12 +11,15 @@ class AgentService(service.MultiService):
 
         director = Director()
 
-        self.web_ui_service = WebUIService(director, web_ui_port)
-        self.web_ui_service.setServiceParent(self)
-
         self.scheduler_service = SchedulerService(director)
         self.scheduler_service.setServiceParent(self)
 
+        self.web_ui_service = WebUIService(director,
+                                           self.scheduler_service,
+                                           web_ui_port)
+        self.web_ui_service.setServiceParent(self)
+
+
     def startService(self):
         service.MultiService.startService(self)
 
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 1f51bd4..74a1688 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -14,6 +14,29 @@ from ooni.contrib import croniter
 from ooni.geoip import probe_ip
 from ooni.measurements import list_measurements
 
+class FileSystemlockAndMutex(object):
+    """
+    This is a lock that is both a mutex lock and also on filesystem.
+    When you acquire it, it will first block on the mutex lock and then
+    once that is released it will attempt to acquire the lock on the
+    filesystem.
+
+    It's a way to support concurrent usage of the DeferredFilesystemLock
+    without races.
+    """
+    def __init__(self, file_path):
+        self._fs_lock = defer.DeferredFilesystemLock(file_path)
+        self._mutex = defer.DeferredLock()
+
+    @defer.inlineCallbacks
+    def acquire(self):
+        yield self._mutex.acquire()
+        yield self._fs_lock.deferUntilLocked()
+
+    def release(self):
+        self._fs_lock.unlock()
+        self._mutex.release()
+
 class DidNotRun(Exception):
     pass
 
@@ -33,7 +56,7 @@ class ScheduledTask(object):
         scheduler_directory = config.scheduler_directory
 
         self._last_run = FilePath(scheduler_directory).child(self.identifier)
-        self._last_run_lock = defer.DeferredFilesystemLock(
+        self._last_run_lock = FileSystemlockAndMutex(
             FilePath(scheduler_directory).child(self.identifier + ".lock").path
         )
 
@@ -63,9 +86,9 @@ class ScheduledTask(object):
 
     @defer.inlineCallbacks
     def run(self):
-        yield self._last_run_lock.deferUntilLocked()
+        yield self._last_run_lock.acquire()
         if not self.should_run:
-            self._last_run_lock.unlock()
+            self._last_run_lock.release()
             raise DidNotRun
         try:
             yield self.task()
@@ -73,7 +96,7 @@ class ScheduledTask(object):
         except:
             raise
         finally:
-            self._last_run_lock.unlock()
+            self._last_run_lock.release()
 
 
 class UpdateInputsAndResources(ScheduledTask):
@@ -140,6 +163,21 @@ class RunDeck(ScheduledTask):
         yield deck.setup()
         yield deck.run(self.director)
 
+
+class RefreshDeckList(ScheduledTask):
+    """
+    This task is configured to refresh the list of decks that are enabled.
+    """
+    identifier = 'refresh-deck-list'
+    schedule = '@hourly'
+
+    def __init__(self, scheduler, schedule=None, identifier=None):
+        self.scheduler = scheduler
+        super(RefreshDeckList, self).__init__(schedule, identifier)
+
+    def task(self):
+        self.scheduler.refresh_deck_list()
+
 class SendHeartBeat(ScheduledTask):
     """
     This task is used to send a heartbeat that the probe is still alive and
@@ -188,6 +226,18 @@ class SchedulerService(service.MultiService):
     def schedule(self, task):
         self._scheduled_tasks.append(task)
 
+    def refresh_deck_list(self):
+        # Deletes all the RunDeck tasks and reschedules only the ones that
+        # are enabled.
+        for scheduled_task in self._scheduled_tasks[:]:
+            if isinstance(scheduled_task, RunDeck):
+                self._scheduled_tasks.remove(scheduled_task)
+
+        for deck_id, deck in deck_store.list_enabled():
+            if deck.schedule is None:
+                continue
+            self.schedule(RunDeck(self.director, deck_id, deck.schedule))
+
     def _task_did_not_run(self, failure, task):
         failure.trap(DidNotRun)
         log.debug("Did not run {0}".format(task.identifier))
@@ -214,13 +264,11 @@ class SchedulerService(service.MultiService):
     def startService(self):
         service.MultiService.startService(self)
 
+        self.refresh_deck_list()
         self.schedule(UpdateInputsAndResources())
         self.schedule(UploadReports())
         self.schedule(DeleteOldReports())
-        for deck_id, deck in deck_store.list_enabled():
-            if deck.schedule is None:
-                continue
-            self.schedule(RunDeck(self.director, deck_id, deck.schedule))
+        self.schedule(RefreshDeckList(self))
 
         self._looping_call.start(self.interval)
 
diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index 1b2300a..75d6366 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -225,7 +225,12 @@ class NGDeck(object):
             measurement_id = task.id
 
             measurement_dir = self._measurement_path.child(measurement_id)
-            measurement_dir.createDirectory()
+            try:
+                measurement_dir.createDirectory()
+            except OSError as ose:
+                # Ignore 'File Exists'
+                if ose.errno != 17:
+                    raise
 
             report_filename = measurement_dir.child("measurements.njson.progress").path
             pid_file = measurement_dir.child("running.pid")
@@ -338,6 +343,8 @@ class DeckTask(object):
 
         if task_data.get('no-collector', False):
             collector_address = None
+        elif config.reports.upload is False:
+            collector_address = None
 
         net_test_loader = NetTestLoader(
             options_to_args(task_data),
diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index 695f97d..2d24f29 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -140,7 +140,7 @@ class DeckStore(object):
     def list_enabled(self):
         decks = []
         for deck_id, deck in self._list():
-            if self.is_enabled(deck_id):
+            if not self.is_enabled(deck_id):
                 continue
             decks.append((deck_id, deck))
         return decks
@@ -153,7 +153,7 @@ class DeckStore(object):
         if not deck_path.exists():
             raise DeckNotFound(deck_id)
         deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
-        deck_enabled_path.linkTo(deck_path)
+        deck_path.linkTo(deck_enabled_path)
 
     def disable(self, deck_id):
         deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
diff --git a/ooni/resources.py b/ooni/resources.py
index aef0f13..9615c53 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -4,6 +4,11 @@ from twisted.python.filepath import FilePath
 from twisted.internet import defer
 from twisted.web.client import downloadPage, getPage, HTTPClientFactory
 
+# WARNING: this script is being run as part of the post install procedure.
+# Be sure to not import either in this module or in the imported modules
+# dependencies other than twisted. If you end up including something that is
+# not twisted, then you will need to add it to the setup_requires in setup.py.
+
 from ooni.utils import log, gunzip, rename
 from ooni.settings import config
 
diff --git a/ooni/scripts/oonideckgen.py b/ooni/scripts/oonideckgen.py
index 9b087f9..6c2882c 100644
--- a/ooni/scripts/oonideckgen.py
+++ b/ooni/scripts/oonideckgen.py
@@ -2,7 +2,6 @@ from __future__ import print_function
 
 import errno
 import os
-import shutil
 import sys
 
 from twisted.internet import defer, task
@@ -29,7 +28,7 @@ class Options(usage.Options):
                                   "submitting reports"],
         ["bouncer", None, None, "Specify a custom bouncer to use"],
         ["output", "o", None,
-         "Specify the directory where to write output."]
+         "Specify the path where we should be writing the deck to."]
     ]
 
     def opt_version(self):
@@ -119,24 +118,13 @@ def oonideckgen(reactor):
         print("%s: --country-code must be 2 characters" % sys.argv[0])
         sys.exit(2)
 
-    if not os.path.isdir(options['output']):
-        print("%s: %s is not a directory" % (sys.argv[0],
-                                             options['output']))
-        sys.exit(3)
+    if os.path.isdir(options['output']):
+        options['output'] = os.path.join(options['output'], 'web-full.yaml')
 
     options['country-code'] = options['country-code'].lower()
 
-    output_dir = os.path.abspath(options['output'])
-    output_dir = os.path.join(output_dir, "deck")
-
-    if os.path.isdir(output_dir):
-        print("Found previous deck deleting content of it")
-        shutil.rmtree(output_dir)
-
-    options['output'] = output_dir
-
     try:
-        os.makedirs(options['output'])
+        os.makedirs(os.path.dirname(options['output']))
     except OSError as exception:
         if exception.errno != errno.EEXIST:
             raise
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py
index 430252a..d67dd80 100644
--- a/ooni/scripts/ooniprobe.py
+++ b/ooni/scripts/ooniprobe.py
@@ -13,7 +13,8 @@ def ooniprobe(reactor):
     if global_options['queue']:
         return runWithDaemonDirector(global_options)
     elif global_options['initialize']:
-        return initializeOoniprobe(global_options)
+        initializeOoniprobe(global_options)
+        return defer.succeed(None)
     elif global_options['web-ui']:
         from ooni.scripts.ooniprobe_agent import WEB_UI_URL
         from ooni.scripts.ooniprobe_agent import status_agent, start_agent
diff --git a/ooni/scripts/oonireport.py b/ooni/scripts/oonireport.py
index 13d8473..cd9b244 100644
--- a/ooni/scripts/oonireport.py
+++ b/ooni/scripts/oonireport.py
@@ -85,8 +85,7 @@ def upload(report_file, collector=None, bouncer=None, measurement_id=None):
             elif isinstance(collector_settings, str):
                 collector_client = CollectorClient(address=collector_settings)
         else:
-            log.msg("Could not find %s in reporting.yaml. Looking up "
-                    "collector with canonical bouncer." % report_file)
+            log.msg("Looking up collector with canonical bouncer." % report_file)
             collector_client = yield lookup_collector_client(report.header,
                                                              CANONICAL_BOUNCER_ONION)
 
@@ -267,8 +266,8 @@ class Options(usage.Options):
 
 def tor_check():
     if not config.tor.socks_port:
-        print("Currently oonireport requires that you start Tor yourself "
-              "and set the socks_port inside of ooniprobe.conf")
+        log.err("Currently oonireport requires that you start Tor yourself "
+                "and set the socks_port inside of ooniprobe.conf")
         sys.exit(1)
 
 
diff --git a/ooni/settings.py b/ooni/settings.py
index 8bb3340..e7174e2 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -13,7 +13,6 @@ from ooni.utils.net import ConnectAndCloseProtocol, connectProtocol
 from ooni.utils import Storage, log, get_ooni_root
 from ooni import errors
 
-
 CONFIG_FILE_TEMPLATE = """\
 # This is the configuration file for OONIProbe
 # This file follows the YAML markup format: http://yaml.org/spec/1.2/spec.html
@@ -127,36 +126,81 @@ defaults = {
         "preferred_backend": "onion"
     },
     "tor": {
+        "socks_port": None,
+        "control_port": None,
+        "bridges": None,
+        "data_dir": None,
         "timeout": 200,
         "torrc": {}
     }
 }
 
+# This is the root of the ooniprobe source code tree
+OONIPROBE_ROOT = get_ooni_root()
+
+IS_VIRTUALENV = False
+if hasattr(sys, 'real_prefix'):
+    IS_VIRTUALENV = True
+
+# These are the the embedded settings
+_SETTINGS_INI = os.path.join(OONIPROBE_ROOT, 'settings.ini')
+
+USR_SHARE_PATH = '/var/lib/ooni'
+VAR_LIB_PATH = '/usr/share/ooni'
+ETC_PATH = '/etc'
+
+if IS_VIRTUALENV:
+    _PREFIX = os.path.abspath(sys.prefix)
+    VAR_LIB_PATH = os.path.join(
+        _PREFIX,
+        'var', 'lib', 'ooni'
+    )
+    USR_SHARE_PATH = os.path.join(
+        _PREFIX,
+        'usr', 'share', 'ooni'
+    )
+    ETC_PATH = os.path.join(
+        _PREFIX,
+        'etc'
+    )
+elif os.path.isfile(_SETTINGS_INI):
+    settings = SafeConfigParser()
+    with open(_SETTINGS_INI) as fp:
+        settings.readfp(fp)
+
+    _USR_SHARE_PATH = settings.get('directories', 'usr_share')
+    if _USR_SHARE_PATH is not None:
+        USR_SHARE_PATH = _USR_SHARE_PATH
+
+    _VAR_LIB_PATH = settings.get('directories', 'var_lib')
+    if _VAR_LIB_PATH is not None:
+        VAR_LIB_PATH = _VAR_LIB_PATH
+
+    _ETC_PATH = settings.get('directories', 'etc')
+    if _ETC_PATH is not None:
+        ETC_PATH = _ETC_PATH
+
 class OConfig(object):
     _custom_home = None
 
     def __init__(self):
         self.current_user = getpass.getuser()
+
         self.global_options = {}
-        self.reports = Storage()
+
         self.scapyFactory = None
         self.tor_state = None
 
         self.logging = True
+
+        # These are the configuration options
         self.basic = Storage()
         self.advanced = Storage()
+        self.reports = Storage()
         self.tor = Storage()
         self.privacy = Storage()
-        self.set_paths()
 
-    def embedded_settings(self, category, option):
-        embedded_settings = os.path.join(get_ooni_root(), 'settings.ini')
-        if os.path.isfile(embedded_settings):
-            settings = SafeConfigParser()
-            with open(embedded_settings) as fp:
-                settings.readfp(fp)
-            return settings.get(category, option)
-        return None
+        self.set_paths()
 
     def is_initialized(self):
         # When this is false it means that the user has not gone
@@ -170,64 +214,23 @@ class OConfig(object):
         with open(initialized_path, 'w+'): pass
 
     @property
-    def var_lib_path(self):
-        if hasattr(sys, 'real_prefix'):
-            # We are in a virtualenv use the /usr/share in the virtualenv
-            return os.path.join(
-                os.path.abspath(sys.prefix),
-                'var', 'lib', 'ooni'
-            )
-        var_lib_path = self.embedded_settings("directories", "var_lib")
-        if var_lib_path:
-            return os.path.abspath(var_lib_path)
-        return "/var/lib/ooni"
-
-    @property
     def running_path(self):
         """
         This is the directory used to store state application data.
         It defaults to /var/lib/ooni, but if that is not writeable we will
         use the ooni_home.
         """
-        var_lib_path = self.var_lib_path
-        if os.access(var_lib_path, os.W_OK):
-            return var_lib_path
+        if os.access(VAR_LIB_PATH, os.W_OK):
+            return VAR_LIB_PATH
         return self.ooni_home
 
     @property
-    def usr_share_path(self):
-        if hasattr(sys, 'real_prefix'):
-            # We are in a virtualenv use the /usr/share in the virtualenv
-            return os.path.join(
-                os.path.abspath(sys.prefix),
-                'usr', 'share', 'ooni'
-            )
-        usr_share_path = self.embedded_settings("directories", "usr_share")
-        if usr_share_path:
-            return os.path.abspath(usr_share_path)
-        return "/usr/share/ooni"
-
-
-    @property
-    def etc_path(self):
-        if hasattr(sys, 'real_prefix'):
-            # We are in a virtualenv use the /usr/share in the virtualenv
-            return os.path.join(
-                os.path.abspath(sys.prefix),
-                'usr', 'share', 'ooni'
-            )
-        etc_path = self.embedded_settings("directories", "etc")
-        if etc_path:
-            return os.path.abspath(etc_path)
-        return "/etc"
-
-    @property
     def data_directory_candidates(self):
         dirs = [
             self.ooni_home,
-            self.var_lib_path,
-            self.usr_share_path,
-            os.path.join(get_ooni_root(), '..', 'data'),
+            VAR_LIB_PATH,
+            USR_SHARE_PATH,
+            os.path.join(OONIPROBE_ROOT, '..', 'data'),
             '/usr/share/'
         ]
         if os.getenv("OONI_DATA_DIR"):
@@ -241,7 +244,7 @@ class OConfig(object):
         for target_dir in self.data_directory_candidates:
             if os.path.isdir(target_dir):
                 return target_dir
-        return self.var_lib_path
+        return VAR_LIB_PATH
 
     @property
     def ooni_home(self):
@@ -260,14 +263,14 @@ class OConfig(object):
                 return file_path
 
     def set_paths(self):
-        self.nettest_directory = os.path.join(get_ooni_root(), 'nettests')
-        self.web_ui_directory = os.path.join(get_ooni_root(), 'ui', 'web', 'client')
+        self.nettest_directory = os.path.join(OONIPROBE_ROOT, 'nettests')
+        self.web_ui_directory = os.path.join(OONIPROBE_ROOT, 'ui', 'web','client')
 
         self.inputs_directory = os.path.join(self.running_path, 'inputs')
         self.scheduler_directory = os.path.join(self.running_path, 'scheduler')
         self.resources_directory = os.path.join(self.running_path, 'resources')
 
-        self.decks_available_directory = os.path.join(self.running_path,
+        self.decks_available_directory = os.path.join(USR_SHARE_PATH,
                                                       'decks-available')
         self.decks_enabled_directory = os.path.join(self.running_path,
                                                     'decks-enabled')
@@ -317,6 +320,7 @@ class OConfig(object):
     def create_config_file(self, include_ip=False, include_asn=True,
                            include_country=True, should_upload=True,
                            preferred_backend="onion"):
+        self.initialize_ooni_home()
         def _bool_to_yaml(value):
             if value is True:
                 return 'true'
@@ -342,39 +346,23 @@ class OConfig(object):
             )
         self.read_config_file()
 
-    def _create_config_file(self):
-        target_config_file = self.config_file
-        print "Creating it for you in '%s'." % target_config_file
-        sample_config_file = self.get_data_file_path('ooniprobe.conf.sample')
-
-        with open(sample_config_file) as f:
-            with open(target_config_file, 'w+') as w:
-                for line in f:
-                    if line.startswith('    logfile: '):
-                        w.write('    logfile: %s\n' % (
-                            os.path.join(self.ooni_home, 'ooniprobe.log'))
-                        )
-                    else:
-                        w.write(line)
-
     def read_config_file(self, check_incoherences=False):
-        #if not os.path.isfile(self.config_file):
-        #    print "Configuration file does not exist."
-        #    self._create_config_file()
-        #    self.read_config_file()
-
         configuration = {}
+        config_file = {}
+        log.debug("Reading config file from %s" % self.config_file)
         if os.path.isfile(self.config_file):
             with open(self.config_file) as f:
                 config_file_contents = '\n'.join(f.readlines())
-                configuration = yaml.safe_load(config_file_contents)
+                config_file = yaml.safe_load(config_file_contents)
 
         for category in defaults.keys():
+            configuration[category] = {}
             for k, v in defaults[category].items():
                 try:
-                    value = configuration.get(category, {})[k]
+                    value = config_file.get(category, {})[k]
                 except KeyError:
                     value = v
+                configuration[category][k] = value
                 getattr(self, category)[k] = value
 
         self.set_paths()
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 8cd3358..65e4a0f 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -10,7 +10,7 @@ from twisted.python import usage
 from twisted.internet import defer
 
 from ooni import errors, __version__
-from ooni.settings import config
+from ooni.settings import config, OONIPROBE_ROOT
 from ooni.utils import log
 
 class LifetimeExceeded(Exception): pass
@@ -182,7 +182,30 @@ def director_startup_other_failures(failure):
 
 
 def initializeOoniprobe(global_options):
-    # XXX print here the informed consent documentation.
+    print("""
+                   _   _              _
+            __ _ _ _ ___ ___| |_(_)_ _  __ _ __| |
+           / _` | '_/ -_) -_)  _| | ' \/ _` (_-<_|
+           \__, |_| \___\___|\__|_|_||_\__, /__(_)
+           |___/                       |___/      )
+          """)
+    print("It looks like this is the first time you are running ooniprobe")
+    print("Please take a minute to read through the informed consent documentation and "
+          "understand what are the risks associated with running ooniprobe.")
+    print("Press enter to continue...")
+    raw_input()
+    with open(os.path.join(OONIPROBE_ROOT, 'ui', 'consent-form.md')) as f:
+        consent_form_text = ''.join(f.readlines())
+    from pydoc import pager
+    pager(consent_form_text)
+
+    answer = ""
+    while answer.lower() != "yes":
+        print('Type "yes" if you are fully aware of the risks associated with using ooniprobe and you wish to proceed')
+        answer = raw_input("> ")
+
+    print("")
+    print("Now help us configure some things!")
     answer = raw_input('Should we upload measurements to a collector? (Y/n) ')
     should_upload = True
     if answer.lower().startswith("n"):
@@ -195,15 +218,15 @@ def initializeOoniprobe(global_options):
 
     answer = raw_input('Should we include your ASN (your network) in '
                        'measurements? (Y/n) ')
-    include_asn = False
+    include_asn = True
     if answer.lower().startswith("n"):
-        include_asn = True
+        include_asn = False
 
     answer = raw_input('Should we include your Country in '
                        'measurements? (Y/n) ')
-    include_country = False
+    include_country = True
     if answer.lower().startswith("n"):
-        include_country = True
+        include_country = False
 
     answer = raw_input('How would you like reports to be uploaded? (onion, '
                        'https, cloudfronted) ')
@@ -230,7 +253,7 @@ def setupGlobalOptions(logging, start_tor, check_incoherences):
         log.err("You first need to agree to the informed consent and setup "
                 "ooniprobe to run it.")
         global_options['initialize'] = True
-        return
+        return global_options
 
     config.set_paths()
     config.initialize_ooni_home()
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index 6a7c149..ad2dc50 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
     <app>
       Loading...
     </app>
-  <script type="text/javascript" src="app.bundle.js?9c4ed560c98eaf61a836"></script></body>
+  <script type="text/javascript" src="app.bundle.js?f06164bd3b339e781c75"></script></body>
 </html>
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index ed2193e..bdafe75 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -149,8 +149,10 @@ class WebUIAPI(object):
     _reactor = reactor
     _enable_xsrf_protection = True
 
-    def __init__(self, config, director, _reactor=reactor):
+    def __init__(self, config, director, scheduler, _reactor=reactor):
         self.director = director
+        self.scheduler = scheduler
+
         self.config = config
         self.measurement_path = FilePath(config.measurements_directory)
 
@@ -301,17 +303,34 @@ class WebUIAPI(object):
         for deck_id, deck in self.director.deck_store.list():
             deck_list['available'][deck_id] = {
                 'name': deck.name,
-                'description': deck.description
+                'description': deck.description,
+                'schedule': deck.schedule,
+                'enabled': self.director.deck_store.is_enabled(deck_id)
             }
 
         for deck_id, deck in self.director.deck_store.list_enabled():
             deck_list['enabled'][deck_id] = {
                 'name': deck.name,
-                'description': deck.description
+                'description': deck.description,
+                'schedule': deck.schedule,
+                'enabled': True
             }
 
         return self.render_json(deck_list, request)
 
+    @app.route('/api/deck/<string:deck_id>/run', methods=["POST"])
+    @xsrf_protect(check=True)
+    @requires_true(attrs=['_director_started', '_is_initialized'])
+    def api_deck_run(self, request, deck_id):
+        try:
+            deck = self.director.deck_store.get(deck_id)
+        except DeckNotFound:
+            raise WebUIError(404, "Deck not found")
+
+        self.run_deck(deck)
+
+        return self.render_json({"status": "starting"}, request)
+
     @app.route('/api/deck/<string:deck_id>/enable', methods=["POST"])
     @xsrf_protect(check=True)
     @requires_true(attrs=['_director_started', '_is_initialized'])
@@ -321,6 +340,8 @@ class WebUIAPI(object):
         except DeckNotFound:
             raise WebUIError(404, "Deck not found")
 
+        self.scheduler.refresh_deck_list()
+
         return self.render_json({"status": "enabled"}, request)
 
     @app.route('/api/deck/<string:deck_id>/disable', methods=["POST"])
@@ -331,6 +352,7 @@ class WebUIAPI(object):
             self.director.deck_store.disable(deck_id)
         except DeckNotFound:
             raise WebUIError(404, "Deck not found")
+        self.scheduler.refresh_deck_list()
 
         return self.render_json({"status": "disabled"}, request)
 
diff --git a/ooni/ui/web/web.py b/ooni/ui/web/web.py
index eca75cb..10bbef1 100644
--- a/ooni/ui/web/web.py
+++ b/ooni/ui/web/web.py
@@ -6,16 +6,17 @@ from ooni.ui.web.server import WebUIAPI
 from ooni.settings import config
 
 class WebUIService(service.MultiService):
-    def __init__(self, director, port_number=8842):
+    def __init__(self, director, scheduler, port_number=8842):
         service.MultiService.__init__(self)
 
         self.director = director
+        self.scheduler = scheduler
         self.port_number = port_number
 
     def startService(self):
         service.MultiService.startService(self)
 
-        web_ui_api = WebUIAPI(config, self.director)
+        web_ui_api = WebUIAPI(config, self.director, self.scheduler)
         self._port = reactor.listenTCP(
             self.port_number,
             server.Site(web_ui_api.app.resource())
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index d672ca8..247758f 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -10,9 +10,6 @@ from zipfile import ZipFile
 from twisted.python.filepath import FilePath
 from twisted.python.runtime import platform
 
-from ooni import errors
-
-
 class Storage(dict):
     """
     A Storage object is like a dictionary except `obj.foo` can be used
@@ -57,6 +54,7 @@ class Storage(dict):
             self[k] = v
 
 def checkForRoot():
+    from ooni import errors
     if os.getuid() != 0:
         raise errors.InsufficientPrivileges
 
diff --git a/setup.py b/setup.py
index da48323..abbbf21 100644
--- a/setup.py
+++ b/setup.py
@@ -92,6 +92,8 @@ import os
 import shutil
 import tempfile
 import subprocess
+from glob import glob
+
 from ConfigParser import SafeConfigParser
 
 from os.path import join as pj
@@ -138,30 +140,20 @@ class OoniInstall(install):
         else:
             var_path = pj(prefix, 'var', 'lib')
 
-        for root, dirs, file_names in os.walk('data/'):
-            files = []
-            for file_name in file_names:
-                if file_name.endswith('.pyc'):
-                    continue
-                elif file_name.endswith('.dat') and \
-                        file_name.startswith('Geo'):
-                    continue
-                elif file_name == "ooniprobe.conf.sample":
-                    files.append(self.gen_config(share_path))
-                    continue
-                files.append(pj(root, file_name))
-            self.distribution.data_files.append(
-                [
-                    pj(share_path, 'ooni', root.replace('data/', '')),
-                    files
-                ]
+        self.distribution.data_files.append(
+            (
+                pj(share_path, 'ooni', 'decks-available'),
+                glob('data/decks/*')
             )
+        )
         settings = SafeConfigParser()
         settings.add_section("directories")
         settings.set("directories", "usr_share",
                      os.path.join(share_path, "ooni"))
         settings.set("directories", "var_lib",
                      os.path.join(var_path, "ooni"))
+        settings.set("directories", "etc",
+                     os.path.join(var_path, "ooni"))
         with open("ooni/settings.ini", "w+") as fp:
             settings.write(fp)
 
@@ -196,7 +188,7 @@ class OoniInstall(install):
         if is_lepidopter():
             self.update_lepidopter_config()
 
-
+setup_requires = ['twisted']
 install_requires = []
 dependency_links = []
 data_files = []
@@ -248,6 +240,7 @@ setup(
     include_package_data=True,
     dependency_links=dependency_links,
     install_requires=install_requires,
+    setup_requires=setup_requires,
     zip_safe=False,
     entry_points={
         'console_scripts': [



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits