[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Add support for deck status



commit c937439d960ba3cfc77d173615fc029c44e417a7
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Wed Jan 11 14:35:03 2017 +0000

    Add support for deck status
---
 ooni/agent/scheduler.py |  2 +-
 ooni/deck/deck.py       | 16 +++++++++++++---
 ooni/deck/store.py      |  3 +--
 ooni/director.py        | 27 +++++++++++++++++++++++++++
 ooni/measurements.py    | 16 +++++++++++-----
 ooni/ui/cli.py          |  2 +-
 ooni/ui/web/server.py   | 38 ++++++++++++++++++--------------------
 7 files changed, 72 insertions(+), 32 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 4bf058ab..8de3b303 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -294,7 +294,7 @@ class RunDeck(ScheduledTask):
     def task(self):
         deck = deck_store.get(self.deck_id)
         yield deck.setup()
-        yield deck.run(self.director)
+        yield deck.run(self.director, from_schedule=True)
 
 
 class RefreshDeckList(ScheduledTask):
diff --git a/ooni/deck/deck.py b/ooni/deck/deck.py
index 07f0799e..fbbb8c3d 100644
--- a/ooni/deck/deck.py
+++ b/ooni/deck/deck.py
@@ -1,6 +1,7 @@
 import os
 import uuid
 import errno
+import hashlib
 from copy import deepcopy
 from string import Template
 
@@ -82,6 +83,7 @@ class NGDeck(object):
         self.name = ""
         self.description = ""
         self.icon = ""
+        self.id = None
         self.schedule = None
 
         self.metadata = {}
@@ -102,10 +104,15 @@ class NGDeck(object):
     def open(self, deck_path, global_options=None):
         with open(deck_path) as fh:
             deck_data = yaml.safe_load(fh)
+        self.id = os.path.basename(deck_path[:-1*len('.yaml')])
         self.deck_directory = os.path.abspath(os.path.dirname(deck_path))
         self.load(deck_data, global_options)
 
     def load(self, deck_data, global_options=None):
+        if self.id is None:
+            # This happens when you load a deck not from a filepath so we
+            # use the first 16 characters of the SHA256 hexdigest as an ID
+            self.id = hashlib.sha256(deck_data).hexdigest()[:16]
         if global_options is not None:
             self.global_options = normalize_options(global_options)
 
@@ -209,7 +216,8 @@ class NGDeck(object):
             )
             generate_summary(
                 measurement_dir.child("measurements.njson").path,
-                measurement_dir.child("summary.json").path
+                measurement_dir.child("summary.json").path,
+                deck_id=self.id
             )
             measurement_dir.child("running.pid").remove()
 
@@ -274,18 +282,20 @@ class NGDeck(object):
         self._is_setup = True
 
     @defer.inlineCallbacks
-    def run(self, director):
+    def run(self, director, from_schedule=False):
         assert self._is_setup, "You must call setup() before you can run a " \
                                "deck"
         if self.requires_tor:
             yield director.start_tor()
         yield self.query_bouncer()
+        director.deckStarted(self.id, from_schedule)
         for task in self._tasks:
             if task.skip is True:
-                log.msg("Skipping running {0}".format(task.id))
+                log.debug("Skipping running {0}".format(task.id))
                 continue
             if task.type == "ooni":
                 yield self._run_ooni_task(task, director)
+        director.deckFinished(self.id, from_schedule)
         self._is_setup = False
 
 
diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index 6e3e6de4..204d9357 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -227,11 +227,10 @@ class DeckStore(object):
         for deck_path in self.available_directory.listdir():
             if not deck_path.endswith('.yaml'):
                 continue
-            deck_id = deck_path[:-1*len('.yaml')]
             deck = NGDeck(
                 deck_path=self.available_directory.child(deck_path).path
             )
-            new_cache[deck_id] = deck
+            new_cache[deck.id] = deck
         self._cache = new_cache
         self._cache_stale = False
 
diff --git a/ooni/director.py b/ooni/director.py
index 1e55bb67..f7566f5f 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -70,6 +70,7 @@ class Director(object):
 
     def __init__(self):
         self.activeNetTests = []
+        self.activeDecks = []
 
         self.measurementManager = MeasurementManager()
         self.measurementManager.director = self
@@ -290,6 +291,32 @@ class Director(object):
         measurement.result = failure
         return measurement
 
+    def deckStarted(self, deck_id, from_schedule):
+        log.debug("Starting {0} ({1})".format(deck_id,
+                                              'scheduled' if from_schedule
+                                              else 'user-run'))
+        self.activeDecks.append((deck_id, from_schedule))
+
+    def deckFinished(self, deck_id, from_schedule):
+        log.debug("Finished {0} ({1})".format(deck_id,
+                                              'scheduled' if from_schedule
+                                              else 'user-run'))
+        try:
+            self.activeDecks.remove((deck_id, from_schedule))
+        except ValueError:
+            log.error("Completed deck {0} is not actually running".format(
+                deck_id))
+
+
+    def isDeckRunning(self, deck_id, from_schedule):
+        """
+        :param deck_id: the ID of the deck to check if it's running
+        :param from_schedule:  True if we want to know the status of a
+        scheduled deck run False for user initiated runs.
+        :return: True if the deck is running False otherwise
+        """
+        return (deck_id, from_schedule) in self.activeDecks
+
     def netTestDone(self, net_test):
         self.notify(DirectorEvent("success",
                                   "Successfully ran test {0}".format(
diff --git a/ooni/measurements.py b/ooni/measurements.py
index b3b3e0fc..9197ee75 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -11,7 +11,7 @@ from ooni.settings import config
 class MeasurementInProgress(Exception):
     pass
 
-class Process():
+class MeasurementTypes():
     supported_tests = [
         "web_connectivity",
         "http_requests",
@@ -51,14 +51,15 @@ class Process():
         result['url'] = entry['input']
         return result
 
-def generate_summary(input_file, output_file):
+
+def generate_summary(input_file, output_file, deck_id='none'):
     results = {}
     with open(input_file) as in_file:
         for idx, line in enumerate(in_file):
             entry = json.loads(line.strip())
             result = {}
-            if entry['test_name'] in Process.supported_tests:
-                result = getattr(Process, entry['test_name'])(entry)
+            if entry['test_name'] in MeasurementTypes.supported_tests:
+                result = getattr(MeasurementTypes, entry['test_name'])(entry)
             result['idx'] = idx
             if not result.get('url', None):
                 result['url'] = entry['input']
@@ -66,6 +67,7 @@ def generate_summary(input_file, output_file):
             results['test_start_time'] = entry['test_start_time']
             results['country_code'] = entry['probe_cc']
             results['asn'] = entry['probe_asn']
+            results['deck_id'] = deck_id
             results['results'] = results.get('results', [])
             results['results'].append(result)
 
@@ -73,9 +75,11 @@ def generate_summary(input_file, output_file):
         json.dump(results, fw)
     return results
 
+
 class MeasurementNotFound(Exception):
     pass
 
+
 def get_measurement(measurement_id, compute_size=False):
     size = -1
     measurement_path = FilePath(config.measurements_directory)
@@ -114,7 +118,9 @@ def get_measurement(measurement_id, compute_size=False):
         "keep": keep,
         "running": running,
         "stale": stale,
-        "size": size
+        "size": size,
+        # XXX we need the deck ID in here
+        "deck_id": "none"
     }
 
 
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 162354c1..cf889a3c 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -377,7 +377,7 @@ def runTestWithDirector(director, global_options, url=None,
     def post_director_start(_):
         try:
             yield deck.setup()
-            yield deck.run(director)
+            yield deck.run(director, from_schedule=False)
         except errors.UnableToLoadDeckInput as error:
             raise defer.failure.Failure(error)
         except errors.NoReachableTestHelpers as error:
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index efcad146..c8f191c7 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -147,7 +147,7 @@ class WebUIAPI(object):
     app = Klein()
     # Maximum number in seconds after which to return a result even if no
     # change happened.
-    _long_polling_timeout = 5
+    _long_polling_timeout = 30
     _reactor = reactor
     _enable_xsrf_protection = True
 
@@ -171,8 +171,8 @@ class WebUIAPI(object):
         # We use exponential backoff to trigger retries of the startup of
         # the director.
         self._director_startup_retries = 0
-        # Maximum delay should be 6 hours
-        self._director_max_retry_delay = 6*60*60
+        # Maximum delay should be 30 minutes
+        self._director_max_retry_delay = 30*60
 
         self.status_poller = LongPoller(
             self._long_polling_timeout, _reactor)
@@ -368,26 +368,21 @@ class WebUIAPI(object):
     @xsrf_protect(check=False)
     @requires_true(attrs=['_director_started', '_is_initialized'])
     def api_deck_list(self, request):
-        deck_list = {
-            'available': {},
-            'enabled': {}
-        }
+        deck_list = {'decks': []}
         for deck_id, deck in self.director.deck_store.list():
-            deck_list['available'][deck_id] = {
+            deck_list['decks'].append({
+                'id': deck_id,
                 'name': deck.name,
+                'icon': deck.icon,
+                'running': self.director.isDeckRunning(
+                                            deck_id, from_schedule=False),
+                'running_scheduled': self.director.isDeckRunning(
+                                            deck_id, from_schedule=True),
+                'nettests': [], #XXX
                 'description': deck.description,
                 'schedule': deck.schedule,
                 'enabled': self.director.deck_store.is_enabled(deck_id)
-            }
-
-        for deck_id, deck in self.director.deck_store.list_enabled():
-            deck_list['enabled'][deck_id] = {
-                'name': deck.name,
-                'description': deck.description,
-                'schedule': deck.schedule,
-                'enabled': True
-            }
-
+            })
         return self.render_json(deck_list, request)
 
     @app.route('/api/deck/<string:deck_id>/run', methods=["POST"])
@@ -433,9 +428,12 @@ class WebUIAPI(object):
         # These are dangling deferreds
         try:
             yield deck.setup()
-            yield deck.run(self.director)
+            yield deck.run(self.director, from_schedule=False)
+            self.director_event_poller.notify(DirectorEvent("success",
+                                                            "Started Deck "
+                                                            + deck.id))
         except:
-            self.director_event_poller.notify(DirectorEvent("error",
+             self.director_event_poller.notify(DirectorEvent("error",
                                                             "Failed to start deck"))
 
     @app.route('/api/nettest/<string:test_name>/start', methods=["POST"])



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits