[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Add script that verifys proper functionality of basic tests



commit 0b9e1629186a081dac4a72046caa8f1244c20154
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Tue Nov 6 16:34:34 2012 +0100

    Add script that verifys proper functionality of basic tests
    * As suggested by Jake
    * There is a subset of inputs that will run with the tests to avoid having to
      wait loads of time for the test running.
    * This should not require more than 30 seconds on a decent network
---
 before_i_commit.sh                        |   27 +++++++++
 ooni/oonicli.py                           |    7 +--
 ooni/utils/log.py                         |   14 +++--
 oonib/oonibackend.py                      |   12 +++-
 oonib/report/api.py                       |   63 ++++------------------
 oonib/report/db/models.py                 |   83 -----------------------------
 test_inputs/README                        |   48 +++++++++++++++++
 test_inputs/dns_tamper_file.txt           |    3 +
 test_inputs/dns_tamper_test_resolvers.txt |    2 +
 test_inputs/http_host_file.txt            |    2 +
 test_inputs/keyword_filtering_file.txt    |    2 +
 test_inputs/url_lists_file.txt            |    2 +
 12 files changed, 118 insertions(+), 147 deletions(-)

diff --git a/before_i_commit.sh b/before_i_commit.sh
new file mode 100755
index 0000000..2816ea5
--- /dev/null
+++ b/before_i_commit.sh
@@ -0,0 +1,27 @@
+#!/bin/sh
+# This script should be run before you commit to verify that the basic tests
+# are working as they should
+# Once you have run it you can inspect the log file via
+#
+# $ less before_i_commit.log
+# To clean up everything that is left by the running of this tool, do as
+# following:
+#
+# rm *.yamloo; rm before_i_commit.log
+#
+
+find . -type f -name "*.py[co]" -delete
+
+./bin/ooniprobe -l before_i_commit.log -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt
+
+./bin/ooniprobe -l before_i_commit.log -o dns_tamper_test.yamloo nettests/core/dnstamper.py -t test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt
+
+./bin/ooniprobe -l before_i_commit.log -o captive_portal_test.yamloo nettests/core/captiveportal.py
+
+./bin/ooniprobe -l before_i_commit.log -o http_host.yamloo nettests/core/http_host.py -b http://ooni.nu/test -f test_inputs/http_host_file.txt
+
+./bin/ooniprobe -l before_i_commit.log -o keyword_filtering.yamloo nettests/core/keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt
+
+./bin/ooniprobe -l before_i_commit.log -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt
+
+
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index aa3cdc0..1988652 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -46,7 +46,7 @@ class Options(usage.Options, app.ReactorSelectionMixin):
 
     optParameters = [
         ["reportfile", "o", None, "report file name"],
-        ["logfile", "l", "test.log", "log file name"],
+        ["logfile", "l", None, "log file name"],
         ['temp-directory', None, '_ooni_temp',
          'Path to use as working directory for tests.']
         ]
@@ -87,9 +87,6 @@ class Options(usage.Options, app.ReactorSelectionMixin):
 
 
 def run():
-    log.start()
-    log.debug("Started logging")
-
     if len(sys.argv) == 1:
         sys.argv.append("--help")
     config = Options()
@@ -101,6 +98,8 @@ def run():
     if config['debug-stacktraces']:
         defer.setDebugging(True)
 
+    log.start(config['logfile'])
+
     classes = runner.findTestClassesFromConfig(config)
     casesList, options = runner.loadTestsAndOptions(classes, config)
 
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index e93295a..771bad8 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -13,18 +13,22 @@ from twisted.python.logfile import DailyLogFile
 from ooni.utils import otime
 from ooni import oconfig
 
-# XXX make this a config option
-log_file = oconfig.basic.logfile
+def start(logfile=None):
+    daily_logfile = None
+
+    if not logfile:
+        logfile = oconfig.basic.logfile
+
+    log_folder = os.path.dirname(logfile)
+    log_filename = os.path.basename(logfile)
 
-def start(log_file=log_file):
-    log_folder = os.path.join('/', *log_file.split('/')[:-1])
-    log_filename = log_file.split('/')[-1]
     daily_logfile = DailyLogFile(log_filename, log_folder)
 
     txlog.msg("Starting OONI on %s (%s UTC)" %  (otime.prettyDateNow(),
                                                  otime.utcPrettyDateNow()))
     logging.basicConfig()
     python_logging = txlog.PythonLoggingObserver()
+
     if oconfig.advanced.debug:
         python_logging.logger.setLevel(logging.DEBUG)
     else:
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 29f3f58..e2e800f 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -9,20 +9,26 @@ import json
 import random
 import string
 
-from twisted.application import internet, service
 from twisted.internet import protocol, reactor, defer
-from twisted.protocols import basic
+from twisted.application import internet, service
 from twisted.web import resource, server, static
 from twisted.web.microdom import escape
+from twisted.protocols import basic
 from twisted.names import dns
 
+from ooni.utils import log
+
 from oonib.report.api import reportingBackend
-from oonib.lib import config
 from oonib.lib.ssl import SSLContext
+from oonib.lib import config
+
 from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer
 from oonib.testhelpers.dns import ProxyDNSServer
 from oonib.testhelpers.daphn3 import Daphn3Server
 
+
+log.start('/tmp/oonib.log')
+
 # This tells twisted to set the
 server.version = config.main.server_version
 
diff --git a/oonib/report/api.py b/oonib/report/api.py
index ad0f62c..ea6085d 100644
--- a/oonib/report/api.py
+++ b/oonib/report/api.py
@@ -8,46 +8,15 @@ This is the async pcap reporting system. It requires the client to have created
 """
 import random
 import string
+import json
+
 from twisted.internet import reactor, defer
+
 from cyclone import web
 
 from oonib.report.db import models
 backend_version = '0.0.1'
 
-def generateReportID():
-    size = 100
-    report_id = ''.join(random.choice(string.ascii_letters) for x in range(size))
-    return report_id
-
-@xxxxxxxxxxxxxxxxxxxxx
-def newReport(software_name, software_version, test_name, test_version,
-               progress, content):
-
-    report_id = generateReportID()
-
-    new_report = models.Report()
-
-    new_report.report_id = unicode(report_id)
-
-    new_report.software_name = unicode(software_name)
-    new_report.software_version = unicode(software_version)
-    new_report.test_name = unicode(test_name)
-    new_report.test_version = unicode(test_version)
-    new_report.progress = unicode(progress)
-    new_report.content = unicode(content)
-
-    print "Software Name: %s" % software_name
-    print "Software Version: %s" % software_version
-    print "Test Name: %s" % test_name
-    print "Test Version: %s" % test_version
-    print "Progress: %s" % progress
-    print "Content: %s" % content
-
-    yield new_report.save()
-
-    defer.returnValue({'backend_version': backend_version, 'report_id':
-                        report_id})
-
 def updateReport(report_id, content):
     print "Report ID: %s" % report_id
     print "Content: %s" % content
@@ -61,7 +30,7 @@ class NewReportHandler(web.RequestHandler):
 
     @web.asynchronous
     @defer.inlineCallbacks
-    def get(self):
+    def post(self):
         """
         Creates a new report with the input
 
@@ -84,27 +53,18 @@ class NewReportHandler(web.RequestHandler):
           {'backend_version': 'XXX', 'report_id': 'XXX'}
 
         """
-        # This is the list of supported arguments
-        arguments = ['software_name', 'software_version',
-                     'test_name','test_version',
-                     'progress', 'content']
-        report = {}
-        for arg in arguments:
-            if len(self.get_arguments(arg)) == 0:
-                raise web.HTTPError(400, "%s not specified as argument of POST"
-                        % arg)
-            report[arg] = self.get_argument(arg)
+        parsed_request = json.loads(self.request.body)
 
-        try:
-            test_helper = self.get_argument('test_helper')
+        # XXX here we should validate and sanitize the request
 
+        report_data = parsed_request
 
-        except web.HTTPError:
-            pass
+        new_report = models.Report()
 
-        new_report = yield newReport(**report)
+        print "Got %s as request" % parsed_request
+        result = yield new_report.new(report_data)
 
-        self.write(new_report)
+        self.write(result)
         self.finish()
 
     def put(self):
@@ -128,4 +88,3 @@ spec = [(r"/report/new", NewReportHandler),
         (r"/report/pcap", PCAPReportHandler)]
 
 reportingBackend = web.Application(spec)
-
diff --git a/oonib/report/db/models.py b/oonib/report/db/models.py
deleted file mode 100644
index 21e60eb..0000000
--- a/oonib/report/db/models.py
+++ /dev/null
@@ -1,83 +0,0 @@
-__all__ = ['Report', 'TestHelperTMP']
-from storm.twisted.transact import transact
-from storm.locals import *
-
-from oonib.report.db import getStore, transactor
-
-class OModel(object):
-
-    transactor = transactor
-
-    @transact
-    def create(query):
-        store = Store(database)
-        store.execute(query)
-        store.commit()
-
-    @transact
-    def save(self):
-        store = getStore()
-        store.add(self)
-        store.commit()
-
-class Report(OModel):
-    """
-    This represents an OONI Report as stored in the database.
-
-    report_id: this is generated by the backend and is used by the client to
-               reference a previous report and append to it. It should be
-               treated as a shared secret between the probe and backend.
-
-    software_name: this indicates the name of the software performing the test
-                   (this will default to ooniprobe)
-
-    software_version: this is the version number of the software running the
-                      test.
-
-    test_name: the name of the test on which the report is being created.
-
-    test_version: indicates the version of the test
-
-    progress: what is the current progress of the report. This allows clients
-              to report event partial reports up to a certain percentage of
-              progress. Once the report is complete progress will be 100.
-
-    content: what is the content of the report. If the current progress is less
-             than 100 we should append to the YAML data structure that is
-             currently stored in such field.
-    """
-    __storm_table__ = 'reports'
-
-    createQuery = "CREATE TABLE " + __storm_table__ +\
-                  "(id INTEGER PRIMARY KEY, report_id VARCHAR, software_name VARCHAR,"\
-                  "software_version VARCHAR, test_name VARCHAR, test_version VARCHAR,"\
-                  "progress VARCHAR, content VARCHAR)"
-
-
-    id = Int(primary=True)
-
-    report_id = Unicode()
-
-    software_name = Unicode()
-    software_version = Unicode()
-    test_name = Unicode()
-    test_version = Unicode()
-    progress = Unicode()
-
-    content = Unicode()
-
-class TestHelperTMP(OModel):
-    __storm_table__ = 'testhelpertmp'
-
-    createQuery = "CREATE TABLE " + __storm_table__ +\
-                  "(id INTEGER PRIMARY KEY, report_id VARCHAR, test_helper VARCHAR,"\
-                  " client_ip VARCHAR, creation_time VARCHAR)"
-
-    id = Int(primary=True)
-
-    report_id = Unicode()
-
-    test_helper = Unicode()
-    client_ip = Unicode()
-
-    creation_time = Date()
diff --git a/test_inputs/README b/test_inputs/README
new file mode 100644
index 0000000..1eff781
--- /dev/null
+++ b/test_inputs/README
@@ -0,0 +1,48 @@
+In here you will find some very simple input lists that are useful for testing
+the correct functionality of the various OONIProbe tests.
+
+# DNS Tamper
+
+./bin/ooniprobe -o dns_tamper_test.yamloo nettests/core/dnstamper.py -t test_inputs/dns_tamper_test_resolvers.txt -f test_inputs/dns_tamper_file.txt
+
+less dns_tamper_test.yamloo
+
+# Captive Portal
+
+./bin/ooniprobe -o captive_portal_test.yamloo nettests/core/captiveportal.py
+
+less captive_portal_test.yamloo
+
+# HTTP Host
+
+./bin/ooniprobe -o http_host.yamloo nettests/core/http_host.py -b http://ooni.nu/test -f test_inputs/http_host_file.txt
+
+less http_host.yamloo
+
+# Keyword filtering
+
+./bin/ooniprobe -o keyword_filtering.yamloo nettests/core/keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt
+
+less keyword_filtering.yamloo
+
+# URL List
+
+./bin/ooniprobe -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt
+
+less url_lists.yamloo
+
+# Squid transparent proxy
+
+./bin/ooniprobe -o squid.yamloo nettests/core/squid.py
+
+less squid.yamloo
+
+# HTTP Requests
+
+Not Implemented
+
+# Traceroute
+
+Not Implemented
+
+
diff --git a/test_inputs/dns_tamper_file.txt b/test_inputs/dns_tamper_file.txt
new file mode 100644
index 0000000..25f365c
--- /dev/null
+++ b/test_inputs/dns_tamper_file.txt
@@ -0,0 +1,3 @@
+torproject.org
+google.com
+measurementlab.net
diff --git a/test_inputs/dns_tamper_test_resolvers.txt b/test_inputs/dns_tamper_test_resolvers.txt
new file mode 100644
index 0000000..14c77e0
--- /dev/null
+++ b/test_inputs/dns_tamper_test_resolvers.txt
@@ -0,0 +1,2 @@
+8.8.8.8
+8.8.4.4
diff --git a/test_inputs/http_host_file.txt b/test_inputs/http_host_file.txt
new file mode 100644
index 0000000..12afb18
--- /dev/null
+++ b/test_inputs/http_host_file.txt
@@ -0,0 +1,2 @@
+torproject.org
+ooni.nu
diff --git a/test_inputs/keyword_filtering_file.txt b/test_inputs/keyword_filtering_file.txt
new file mode 100644
index 0000000..4583bae
--- /dev/null
+++ b/test_inputs/keyword_filtering_file.txt
@@ -0,0 +1,2 @@
+antani
+sblinda
diff --git a/test_inputs/url_lists_file.txt b/test_inputs/url_lists_file.txt
new file mode 100644
index 0000000..16a4f58
--- /dev/null
+++ b/test_inputs/url_lists_file.txt
@@ -0,0 +1,2 @@
+http://ooni.nu/test
+http://torproject.org/

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits