[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Add support for listing enabled and disabled decks
commit 7c35e65485418d1dcb682bad1cf4eb62ef318e81
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date: Sat Jul 30 16:27:20 2016 +0200
Add support for listing enabled and disabled decks
* Fix various bugs
---
ooni/agent/scheduler.py | 2 +-
ooni/deck/store.py | 40 +++++++++++++++++----
ooni/director.py | 5 +--
ooni/measurements.py | 11 ++++--
ooni/settings.py | 9 +++--
ooni/ui/web/client/index.html | 2 +-
ooni/ui/web/server.py | 81 +++++++++++++++++++++++++++++++++----------
7 files changed, 117 insertions(+), 33 deletions(-)
diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 43715e8..7a77afb 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -131,7 +131,7 @@ class RunDecks(ScheduledTask):
@defer.inlineCallbacks
def task(self):
- for deck_id, deck in deck_store.list():
+ for deck_id, deck in deck_store.list_enabled():
yield deck.setup()
yield deck.run(self.director)
diff --git a/ooni/deck/store.py b/ooni/deck/store.py
index 7c90204..695f97d 100644
--- a/ooni/deck/store.py
+++ b/ooni/deck/store.py
@@ -9,7 +9,6 @@ from ooni.deck.deck import NGDeck
from ooni.otime import timestampNowISO8601UTC
from ooni.resources import check_for_update
from ooni.settings import config
-from ooni.utils import log
class InputNotFound(Exception):
pass
@@ -121,25 +120,54 @@ class InputStore(object):
class DeckStore(object):
def __init__(self):
- self.path = FilePath(config.decks_directory)
+ self.enabled_directory = FilePath(config.decks_enabled_directory)
+ self.available_directory = FilePath(config.decks_available_directory)
self._cache = {}
self._cache_stale = True
- def list(self):
- decks = []
+ def _list(self):
if self._cache_stale:
self._update_cache()
for deck_id, deck in self._cache.iteritems():
+ yield (deck_id, deck)
+
+ def list(self):
+ decks = []
+ for deck_id, deck in self._list():
decks.append((deck_id, deck))
return decks
+ def list_enabled(self):
+ decks = []
+ for deck_id, deck in self._list():
+ if self.is_enabled(deck_id):
+ continue
+ decks.append((deck_id, deck))
+ return decks
+
+ def is_enabled(self, deck_id):
+ return self.enabled_directory.child(deck_id + '.yaml').exists()
+
+ def enable(self, deck_id):
+ deck_path = self.available_directory.child(deck_id + '.yaml')
+ if not deck_path.exists():
+ raise DeckNotFound(deck_id)
+ deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
+ deck_enabled_path.linkTo(deck_path)
+
+ def disable(self, deck_id):
+ deck_enabled_path = self.enabled_directory.child(deck_id + '.yaml')
+ if not deck_enabled_path.exists():
+ raise DeckNotFound(deck_id)
+ deck_enabled_path.remove()
+
def _update_cache(self):
- for deck_path in self.path.listdir():
+ for deck_path in self.available_directory.listdir():
if not deck_path.endswith('.yaml'):
continue
deck_id = deck_path[:-1*len('.yaml')]
deck = NGDeck(
- deck_path=self.path.child(deck_path).path
+ deck_path=self.available_directory.child(deck_path).path
)
self._cache[deck_id] = deck
diff --git a/ooni/director.py b/ooni/director.py
index 464a203..304a14a 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -9,7 +9,7 @@ from ooni.utils import log, generate_filename
from ooni.nettest import NetTest, getNetTestInformation
from ooni.settings import config
from ooni.nettest import normalizeTestName
-from ooni.deck.store import InputStore
+from ooni.deck.store import input_store, deck_store
from ooni.geoip import probe_ip
from ooni.agent.scheduler import run_system_tasks
@@ -96,7 +96,8 @@ class Director(object):
self.allTestsDone = defer.Deferred()
self.sniffers = {}
- self.input_store = InputStore()
+ self.input_store = input_store
+ self.deck_store = deck_store
self._reset_director_state()
self._reset_tor_state()
diff --git a/ooni/measurements.py b/ooni/measurements.py
index 50efd87..cdf6b14 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -5,6 +5,9 @@ import signal
from twisted.python.filepath import FilePath
from ooni.settings import config
+class MeasurementInProgress(Exception):
+ pass
+
class Process():
supported_tests = [
"web_connectivity",
@@ -66,7 +69,7 @@ def get_measurement(measurement_id):
running = False
completed = True
keep = False
- if measurement.child("measurement.njson.progress").exists():
+ if measurement.child("measurements.njson.progress").exists():
completed = False
# XXX this is done quite often around the code, probably should
# be moved into some utility function.
@@ -97,10 +100,14 @@ def get_measurement(measurement_id):
def get_summary(measurement_id):
measurement_path = FilePath(config.measurements_directory)
measurement = measurement_path.child(measurement_id)
+
+ if measurement.child("measurements.njson.progress").exists():
+ raise MeasurementInProgress
+
summary = measurement.child("summary.json")
if not summary.exists():
generate_summary(
- measurement.child("measurement.njson").path,
+ measurement.child("measurements.njson").path,
summary.path
)
diff --git a/ooni/settings.py b/ooni/settings.py
index 3d33601..2161560 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -135,9 +135,13 @@ class OConfig(object):
self.inputs_directory = os.path.join(self.running_path, 'inputs')
self.scheduler_directory = os.path.join(self.running_path, 'scheduler')
- self.decks_directory = os.path.join(self.running_path, 'decks')
self.resources_directory = os.path.join(self.running_path, 'resources')
+ self.decks_available_directory = os.path.join(self.running_path,
+ 'decks-available')
+ self.decks_enabled_directory = os.path.join(self.running_path,
+ 'decks-enabled')
+
self.measurements_directory = os.path.join(self.running_path,
'measurements')
@@ -166,7 +170,8 @@ class OConfig(object):
# also ensure the subdirectories exist
sub_directories = [
self.inputs_directory,
- self.decks_directory,
+ self.decks_enabled_directory,
+ self.decks_available_directory,
self.scheduler_directory,
self.measurements_directory,
self.resources_directory
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index ebed106..e363ba0 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
<app>
Loading...
</app>
- <script type="text/javascript" src="app.bundle.js?2777836bc218e75c3be5"></script></body>
+ <script type="text/javascript" src="app.bundle.js?9d3ccb3bc67af5ed4453"></script></body>
</html>
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 74cb235..50db3ad 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -1,6 +1,5 @@
from __future__ import print_function
-import os
import json
import string
from functools import wraps
@@ -17,11 +16,12 @@ from werkzeug.exceptions import NotFound
from ooni import __version__ as ooniprobe_version
from ooni import errors
from ooni.deck import NGDeck
+from ooni.deck.store import DeckNotFound, InputNotFound
from ooni.settings import config
from ooni.utils import log
from ooni.director import DirectorEvent
-from ooni.measurements import get_summary, get_measurement
-from ooni.measurements import list_measurements, MeasurementNotFound
+from ooni.measurements import get_summary, get_measurement, list_measurements
+from ooni.measurements import MeasurementNotFound, MeasurementInProgress
from ooni.geoip import probe_ip
config.advanced.debug = True
@@ -120,7 +120,6 @@ class WebUIAPI(object):
self.director = director
self.config = config
self.measurement_path = FilePath(config.measurements_directory)
- self.decks_path = FilePath(config.decks_directory)
# We use a double submit token to protect against XSRF
rng = SystemRandom()
@@ -207,23 +206,61 @@ class WebUIAPI(object):
d.addCallback(got_status_update)
return d
- @app.route('/api/deck/generate', methods=["GET"])
- @xsrf_protect(check=False)
- def api_deck_generate(self, request):
- return self.render_json({"generate": "deck"}, request)
-
- @app.route('/api/deck/<string:deck_name>/start', methods=["POST"])
+ @app.route('/api/deck/<string:deck_id>/start', methods=["POST"])
@xsrf_protect(check=True)
- def api_deck_start(self, request, deck_name):
- return self.render_json({"start": deck_name}, request)
+ def api_deck_start(self, request, deck_id):
+ try:
+ deck = self.director.deck_store.get(deck_id)
+ except DeckNotFound:
+ raise WebUIError(404, "Deck not found")
+
+ try:
+ self.run_deck(deck)
+ except:
+ raise WebUIError(500, "Failed to start deck")
+
+ return self.render_json({"status": "started " + deck.name}, request)
@app.route('/api/deck', methods=["GET"])
@xsrf_protect(check=False)
def api_deck_list(self, request):
- for deck_id in self.decks_path.listdir():
- pass
+ deck_list = {
+ 'available': {},
+ 'enabled': {}
+ }
+ for deck_id, deck in self.director.deck_store.list():
+ deck_list['available'][deck_id] = {
+ 'name': deck.name,
+ 'description': deck.description
+ }
+
+ for deck_id, deck in self.director.deck_store.list_enabled():
+ deck_list['enabled'][deck_id] = {
+ 'name': deck.name,
+ 'description': deck.description
+ }
+
+ return self.render_json(deck_list, request)
- return self.render_json({"command": "deck-list"}, request)
+ @app.route('/api/deck/<string:deck_id>/enable', methods=["POST"])
+ @xsrf_protect(check=True)
+ def api_deck_enable(self, request, deck_id):
+ try:
+ self.director.deck_store.enable(deck_id)
+ except DeckNotFound:
+ raise WebUIError(404, "Deck not found")
+
+ return self.render_json({"status": "enabled"}, request)
+
+ @app.route('/api/deck/<string:deck_id>/disable', methods=["POST"])
+ @xsrf_protect(check=True)
+ def api_deck_disable(self, request, deck_id):
+ try:
+ self.director.deck_store.disable(deck_id)
+ except DeckNotFound:
+ raise WebUIError(404, "Deck not found")
+
+ return self.render_json({"status": "disabled"}, request)
@defer.inlineCallbacks
def run_deck(self, deck):
@@ -261,17 +298,21 @@ class WebUIAPI(object):
except errors.MissingRequiredOption, option_name:
raise WebUIError(
- 501, 'Missing required option: "{}"'.format(option_name)
+ 400, 'Missing required option: "{}"'.format(option_name)
)
except usage.UsageError:
raise WebUIError(
- 502, 'Error in parsing options'
+ 400, 'Error in parsing options'
)
except errors.InsufficientPrivileges:
raise WebUIError(
- 502, 'Insufficient priviledges'
+ 400, 'Insufficient priviledges'
+ )
+ except:
+ raise WebUIError(
+ 500, 'Failed to start nettest'
)
return self.render_json({"status": "started"}, request)
@@ -319,6 +360,8 @@ class WebUIAPI(object):
raise WebUIError(500, "invalid measurement id")
except MeasurementNotFound:
raise WebUIError(404, "measurement not found")
+ except MeasurementInProgress:
+ raise WebUIError(400, "measurement in progress")
if measurement['completed'] is False:
raise WebUIError(400, "measurement in progress")
@@ -359,7 +402,7 @@ class WebUIAPI(object):
with summary.open("w+") as f:
pass
- return self.render_json({"result": "ok"}, request)
+ return self.render_json({"status": "ok"}, request)
@app.route('/api/measurement/<string:measurement_id>/<int:idx>',
methods=["GET"])
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits