[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Add commands for managing the lifecycle of the ooniprobe-agent



commit fe64930f836dccf76a87b9e9bcfe655bada49f90
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Fri Jul 29 16:47:32 2016 +0200

    Add commands for managing the lifecycle of the ooniprobe-agent
    
    * Add command to start the web-ui from ooniprobe
    * Improvements to the measurements API
    * Add klein dependency
---
 ooni/agent/agent.py             |   4 +-
 ooni/measurements.py            |  93 ++++++++++++++++++++++------
 ooni/reporter.py                |  13 +++-
 ooni/scripts/ooniprobe.py       |  15 ++++-
 ooni/scripts/ooniprobe_agent.py | 130 +++++++++++++++++++++++++++++++++++++++-
 ooni/ui/cli.py                  |   5 +-
 ooni/ui/web/client/index.html   |   2 +-
 ooni/ui/web/server.py           |  53 +++++-----------
 requirements.txt                |   1 +
 9 files changed, 247 insertions(+), 69 deletions(-)

diff --git a/ooni/agent/agent.py b/ooni/agent/agent.py
index f5322ed..ff9a2dd 100644
--- a/ooni/agent/agent.py
+++ b/ooni/agent/agent.py
@@ -7,7 +7,7 @@ from ooni.ui.web.web import WebUIService
 from ooni.agent.scheduler import SchedulerService
 
 class AgentService(service.MultiService):
-    def __init__(self):
+    def __init__(self, web_ui_port):
         service.MultiService.__init__(self)
 
         director = Director()
@@ -15,7 +15,7 @@ class AgentService(service.MultiService):
         config.initialize_ooni_home()
         config.read_config_file()
 
-        self.web_ui_service = WebUIService(director)
+        self.web_ui_service = WebUIService(director, web_ui_port)
         self.web_ui_service.setServiceParent(self)
 
         self.scheduler_service = SchedulerService(director)
diff --git a/ooni/measurements.py b/ooni/measurements.py
index f830a99..50efd87 100644
--- a/ooni/measurements.py
+++ b/ooni/measurements.py
@@ -1,10 +1,14 @@
+import os
 import json
+import signal
+
 from twisted.python.filepath import FilePath
 from ooni.settings import config
 
 class Process():
     supported_tests = [
-        "web_connectivity"
+        "web_connectivity",
+        "http_requests"
     ]
     @staticmethod
     def web_connectivity(entry):
@@ -15,6 +19,22 @@ class Process():
         result['url'] = entry['input']
         return result
 
+    @staticmethod
+    def http_requests(entry):
+        result = {}
+        test_keys = entry['test_keys']
+        anomaly = (
+            test_keys['body_length_match'] and
+            test_keys['headers_match'] and
+            (
+                test_keys['control_failure'] !=
+                test_keys['experiment_failure']
+            )
+        )
+        result['anomaly'] = anomaly
+        result['url'] = entry['input']
+        return result
+
 def generate_summary(input_file, output_file):
     results = {}
     with open(input_file) as in_file:
@@ -34,29 +54,64 @@ def generate_summary(input_file, output_file):
     with open(output_file, "w") as fw:
         json.dump(results, fw)
 
+class MeasurementNotFound(Exception):
+    pass
+
+def get_measurement(measurement_id):
+    measurement_path = FilePath(config.measurements_directory)
+    measurement = measurement_path.child(measurement_id)
+    if not measurement.exists():
+        raise MeasurementNotFound
+
+    running = False
+    completed = True
+    keep = False
+    if measurement.child("measurement.njson.progress").exists():
+        completed = False
+        # XXX this is done quite often around the code, probably should
+        # be moved into some utility function.
+        pid = measurement.child("running.pid").open("r").read()
+        pid = int(pid)
+        try:
+            os.kill(pid, signal.SIG_DFL)
+            running = True
+        except OSError:
+            pass
+
+    if measurement.child("keep").exists():
+        keep = True
+    test_start_time, country_code, asn, test_name = \
+        measurement_id.split("-")[:4]
+    return {
+        "test_name": test_name,
+        "country_code": country_code,
+        "asn": asn,
+        "test_start_time": test_start_time,
+        "id": measurement_id,
+        "completed": completed,
+        "keep": keep,
+        "running": running
+    }
+
+
+def get_summary(measurement_id):
+    measurement_path = FilePath(config.measurements_directory)
+    measurement = measurement_path.child(measurement_id)
+    summary = measurement.child("summary.json")
+    if not summary.exists():
+        generate_summary(
+            measurement.child("measurement.njson").path,
+            summary.path
+        )
+
+    with summary.open("r") as f:
+        return json.load(f)
 
 def list_measurements():
     measurements = []
     measurement_path = FilePath(config.measurements_directory)
     for measurement_id in measurement_path.listdir():
-        measurement = measurement_path.child(measurement_id)
-        completed = True
-        keep = False
-        if measurement.child("measurement.njson.progress").exists():
-            completed = False
-        if measurement.child("keep").exists():
-            keep = True
-        test_start_time, country_code, asn, test_name = \
-            measurement_id.split("-")[:4]
-        measurements.append({
-            "test_name": test_name,
-            "country_code": country_code,
-            "asn": asn,
-            "test_start_time": test_start_time,
-            "id": measurement_id,
-            "completed": completed,
-            "keep": keep
-        })
+        measurements.append(get_measurement(measurement_id))
     return measurements
 
 if __name__ == "__main__":
diff --git a/ooni/reporter.py b/ooni/reporter.py
index cf5341d..cfdce08 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -6,7 +6,6 @@ import os
 from copy import deepcopy
 
 from datetime import datetime
-from contextlib import contextmanager
 
 from yaml.representer import SafeRepresenter
 from yaml.emitter import Emitter
@@ -36,6 +35,7 @@ from ooni.utils import generate_filename
 from ooni.settings import config
 
 from ooni.tasks import ReportEntry
+from ooni.measurements import list_measurements
 
 
 def createPacketReport(packet_list):
@@ -413,9 +413,10 @@ class OONIBReportLog(object):
     @defer.inlineCallbacks
     def get_report_log_entries(self):
         entries = []
-        for measurement_id in self.measurement_dir.listdir():
+        for measurement in list_measurements():
             try:
-                entry = yield self.get_report_log(measurement_id)
+                entry = yield self.get_report_log(measurement['id'])
+                entry['completed'] = measurement['completed']
                 entries.append(entry)
             except NoReportLog:
                 continue
@@ -453,6 +454,9 @@ class OONIBReportLog(object):
         incomplete_reports = []
         all_entries = yield self.get_report_log_entries()
         for entry in all_entries[:]:
+            # This means that the measurement itself is incomplete
+            if entry['completed'] is False:
+                continue
             if entry['status'] in ('created',):
                 try:
                     os.kill(entry['pid'], 0)
@@ -486,6 +490,9 @@ class OONIBReportLog(object):
         to_upload_reports = []
         all_entries = yield self.get_report_log_entries()
         for entry in all_entries[:]:
+            # This means that the measurement itself is incomplete
+            if entry['completed'] is False:
+                continue
             if entry['status'] in ('creation-failed', 'not-created'):
                 to_upload_reports.append(
                     (entry['measurements_path'], entry)
diff --git a/ooni/scripts/ooniprobe.py b/ooni/scripts/ooniprobe.py
index 24493da..f5d5b59 100644
--- a/ooni/scripts/ooniprobe.py
+++ b/ooni/scripts/ooniprobe.py
@@ -1,5 +1,8 @@
 #!/usr/bin/env python
-from twisted.internet import task
+import webbrowser
+from multiprocessing import Process
+
+from twisted.internet import task, defer
 
 def ooniprobe(reactor):
     from ooni.ui.cli import runWithDaemonDirector, runWithDirector
@@ -9,6 +12,16 @@ def ooniprobe(reactor):
                                         check_incoherences=True)
     if global_options['queue']:
         return runWithDaemonDirector(global_options)
+    elif global_options['web-ui']:
+        from ooni.scripts.ooniprobe_agent import WEB_UI_URL
+        from ooni.scripts.ooniprobe_agent import status_agent, start_agent
+        if status_agent() != 0:
+            p = Process(target=start_agent)
+            p.start()
+            p.join()
+            print("Started ooniprobe-agent")
+        webbrowser.open_new(WEB_UI_URL)
+        return defer.succeed(None)
     else:
         return runWithDirector(global_options)
 
diff --git a/ooni/scripts/ooniprobe_agent.py b/ooni/scripts/ooniprobe_agent.py
index 7833308..3f8efc8 100644
--- a/ooni/scripts/ooniprobe_agent.py
+++ b/ooni/scripts/ooniprobe_agent.py
@@ -1,22 +1,58 @@
+from __future__ import print_function
+
+import os
+import time
+import signal
+
 from twisted.scripts import twistd
 from twisted.python import usage
 
+from ooni.settings import config
 from ooni.agent.agent import AgentService
 
+WEB_UI_PORT = 8842
+WEB_UI_URL = "http://127.0.0.1:{0}".format(WEB_UI_PORT)
+
 class StartOoniprobeAgentPlugin:
     tapname = "ooniprobe"
 
     def makeService(self, so):
-        return AgentService()
+        return AgentService(WEB_UI_PORT)
 
 class OoniprobeTwistdConfig(twistd.ServerOptions):
     subCommands = [
         ("StartOoniprobeAgent", None, usage.Options, "ooniprobe agent")
     ]
 
-def run():
-    twistd_args = ["--nodaemon"]
+class StartOptions(usage.Options):
+    pass
+
+class StopOptions(usage.Options):
+    pass
+
+class StatusOptions(usage.Options):
+    pass
+
+class RunOptions(usage.Options):
+    pass
+
+class AgentOptions(usage.Options):
+    subCommands = [
+        ['start', None, StartOptions, "Start the ooniprobe-agent in the "
+                                      "background"],
+        ['stop', None, StopOptions, "Stop the ooniprobe-agent"],
+        ['status', None, StatusOptions, "Show status of the ooniprobe-agent"],
+        ['run', None, RunOptions, "Run the ooniprobe-agent in the foreground"]
+    ]
+    def postOptions(self):
+        self.twistd_args = []
+
+def start_agent(options=None):
+    os.chdir(config.ooni_home)
+    twistd_args = []
     twistd_config = OoniprobeTwistdConfig()
+    if options is not None:
+        twistd_args.extend(options.twistd_args)
     twistd_args.append("StartOoniprobeAgent")
     try:
         twistd_config.parseOptions(twistd_args)
@@ -25,8 +61,96 @@ def run():
     twistd_config.loadedPlugins = {
         "StartOoniprobeAgent": StartOoniprobeAgentPlugin()
     }
+    print("Starting ooniprobe agent.")
+    print("To view the GUI go to %s" % WEB_UI_URL)
     twistd.runApp(twistd_config)
     return 0
 
+def status_agent():
+    pidfile = os.path.join(
+        config.ooni_home,
+        'twistd.pid'
+    )
+    if not os.path.exists(pidfile):
+        print("ooniprobe-agent is NOT running")
+        return  1
+    pid = open(pidfile, "r").read()
+    pid = int(pid)
+    try:
+        os.kill(pid, signal.SIG_DFL)
+    except OSError, oserr:
+        if oserr.errno == 3:
+            print("ooniprobe-agent is NOT running")
+            return 1
+    print("ooniprobe-agent is running")
+    return 0
+
+def stop_agent():
+    # This function is borrowed from tahoe
+    pidfile = os.path.join(
+        config.ooni_home,
+        'twistd.pid'
+    )
+    if not os.path.exists(pidfile):
+        print("It seems like ooniprobe-agent is not running")
+        return 2
+    pid = open(pidfile, "r").read()
+    pid = int(pid)
+    try:
+        os.kill(pid, signal.SIGKILL)
+    except OSError, oserr:
+        if oserr.errno == 3:
+            print("No process was running. Cleaning up.")
+            # the process didn't exist, so wipe the pid file
+            os.remove(pidfile)
+            return 2
+        else:
+            raise
+    try:
+        os.remove(pidfile)
+    except EnvironmentError:
+        pass
+    start = time.time()
+    time.sleep(0.1)
+    wait = 40
+    first_time = True
+    while True:
+        # poll once per second until we see the process is no longer running
+        try:
+            os.kill(pid, 0)
+        except OSError:
+            print("process %d is dead" % pid)
+            return
+        wait -= 1
+        if wait < 0:
+            if first_time:
+                print("It looks like pid %d is still running "
+                      "after %d seconds" % (pid, (time.time() - start)))
+                print("I will keep watching it until you interrupt me.")
+                wait = 10
+                first_time = False
+            else:
+                print("pid %d still running after %d seconds" % \
+                (pid, (time.time() - start)))
+                wait = 10
+        time.sleep(1)
+    # we define rc=1 to mean "I think something is still running, sorry"
+    return 1
+
+def run():
+    options = AgentOptions()
+    options.parseOptions()
+
+    if options.subCommand == "run":
+        options.twistd_args += ("--nodaemon",)
+
+    if options.subCommand == "stop":
+        return stop_agent()
+
+    if options.subCommand == "status":
+        return status_agent()
+
+    return start_agent(options)
+
 if __name__ == "__main__":
     run()
diff --git a/ooni/ui/cli.py b/ooni/ui/cli.py
index 2b5d844..3eccf9a 100644
--- a/ooni/ui/cli.py
+++ b/ooni/ui/cli.py
@@ -29,7 +29,8 @@ class Options(usage.Options):
                 ["no-geoip", "g", "Disable geoip lookup on start"],
                 ["list", "s", "List the currently installed ooniprobe "
                               "nettests"],
-                ["verbose", "v", "Show more verbose information"]
+                ["verbose", "v", "Show more verbose information"],
+                ["web-ui", "w", "Start the web UI"]
                 ]
 
     optParameters = [
@@ -96,7 +97,7 @@ This will tell you how to run ooniprobe :)
         sys.exit(0)
 
     def parseArgs(self, *args):
-        if self['testdeck'] or self['list']:
+        if self['testdeck'] or self['list'] or self['web-ui']:
             return
         try:
             self['test_file'] = args[0]
diff --git a/ooni/ui/web/client/index.html b/ooni/ui/web/client/index.html
index 7ba33fb..ebed106 100644
--- a/ooni/ui/web/client/index.html
+++ b/ooni/ui/web/client/index.html
@@ -13,5 +13,5 @@
     <app>
       Loading...
     </app>
-  <script type="text/javascript" src="app.bundle.js?afba2f26969b4c8f00ec"></script></body>
+  <script type="text/javascript" src="app.bundle.js?2777836bc218e75c3be5"></script></body>
 </html>
diff --git a/ooni/ui/web/server.py b/ooni/ui/web/server.py
index 9d98e62..e1a6398 100644
--- a/ooni/ui/web/server.py
+++ b/ooni/ui/web/server.py
@@ -20,7 +20,8 @@ from ooni.deck import NGDeck
 from ooni.settings import config
 from ooni.utils import log
 from ooni.director import DirectorEvent
-from ooni.measurements import generate_summary
+from ooni.measurements import get_summary, get_measurement
+from ooni.measurements import list_measurements, MeasurementNotFound
 from ooni.geoip import probe_ip
 
 config.advanced.debug = True
@@ -311,61 +312,40 @@ class WebUIAPI(object):
     @app.route('/api/measurement', methods=["GET"])
     @xsrf_protect(check=False)
     def api_measurement_list(self, request):
-        measurements = []
-        for measurement_id in self.measurement_path.listdir():
-            measurement = self.measurement_path.child(measurement_id)
-            completed = True
-            if measurement.child("measurement.njson.progress").exists():
-                completed = False
-            test_start_time, country_code, asn, test_name = \
-                measurement_id.split("-")[:4]
-            measurements.append({
-                "test_name": test_name,
-                "country_code": country_code,
-                "asn": asn,
-                "test_start_time": test_start_time,
-                "id": measurement_id,
-                "completed": completed
-            })
+        measurements = list_measurements()
         return self.render_json({"measurements": measurements}, request)
 
     @app.route('/api/measurement/<string:measurement_id>', methods=["GET"])
     @xsrf_protect(check=False)
     def api_measurement_summary(self, request, measurement_id):
         try:
-            measurement_dir = self.measurement_path.child(measurement_id)
+            measurement = get_measurement(measurement_id)
         except InsecurePath:
             raise WebUIError(500, "invalid measurement id")
+        except MeasurementNotFound:
+            raise WebUIError(404, "measurement not found")
 
-        if measurement_dir.child("measurements.njson.progress").exists():
-            raise WebUIError(400, "measurement in progress")
-
-        if not measurement_dir.child("summary.json").exists():
-            # XXX we can perhaps remove this.
-            generate_summary(
-                measurement_dir.child("measurements.njson").path,
-                measurement_dir.child("summary.json").path
-            )
+        if measurement['completed'] is False:
             raise WebUIError(400, "measurement in progress")
 
-        summary = measurement_dir.child("summary.json")
-        with summary.open("r") as f:
-            r = json.load(f)
-
-        return self.render_json(r, request)
+        summary = get_summary(measurement_id)
+        return self.render_json(summary, request)
 
     @app.route('/api/measurement/<string:measurement_id>', methods=["DELETE"])
     @xsrf_protect(check=True)
     def api_measurement_delete(self, request, measurement_id):
         try:
-            measurement_dir = self.measurement_path.child(measurement_id)
+            measurement = get_measurement(measurement_id)
         except InsecurePath:
             raise WebUIError(500, "invalid measurement id")
+        except MeasurementNotFound:
+            raise WebUIError(404, "measurement not found")
 
-        if measurement_dir.child("measurements.njson.progress").exists():
-            raise WebUIError(400, "measurement in progress")
+        if measurement['running'] is True:
+            raise WebUIError(400, "Measurement running")
 
         try:
+            measurement_dir = self.measurement_path.child(measurement_id)
             measurement_dir.remove()
         except:
             raise WebUIError(400, "Failed to delete report")
@@ -380,9 +360,6 @@ class WebUIAPI(object):
         except InsecurePath:
             raise WebUIError(500, "invalid measurement id")
 
-        if measurement_dir.child("measurements.njson.progress").exists():
-            raise WebUIError(400, "measurement in progress")
-
         summary = measurement_dir.child("keep")
         with summary.open("w+") as f:
             pass
diff --git a/requirements.txt b/requirements.txt
index 410566e..c05ebcb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -17,3 +17,4 @@ service-identity
 pydumbnet
 zope.interface
 certifi
+klein



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits