[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Now we have the database properly hooked up to OONIB



commit b1ca4257c13dff18664c8b175ed6b57acc8874b3
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Fri Oct 5 10:58:57 2012 +0000

    Now we have the database properly hooked up to OONIB
    * Support writing to DB of data relative to a report
---
 oonib/oonibackend.conf.sample |    1 +
 oonib/oonibackend.py          |   36 ++++++++++++++-----
 oonib/report/api.py           |   75 +++++++++++++++++++++++++++++-----------
 oonib/report/db/__init__.py   |   20 +++++-----
 oonib/report/db/models.py     |   44 +++++++++++++++++++++++-
 5 files changed, 134 insertions(+), 42 deletions(-)

diff --git a/oonib/oonibackend.conf.sample b/oonib/oonibackend.conf.sample
index 021e0e2..dc0a662 100644
--- a/oonib/oonibackend.conf.sample
+++ b/oonib/oonibackend.conf.sample
@@ -1,4 +1,5 @@
 [main]
+reporting_port = 8888
 http_port = 8080
 dns_udp_port = 5354
 dns_tcp_port = 8002
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 0232829..29f3f58 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -16,6 +16,7 @@ from twisted.web import resource, server, static
 from twisted.web.microdom import escape
 from twisted.names import dns
 
+from oonib.report.api import reportingBackend
 from oonib.lib import config
 from oonib.lib.ssl import SSLContext
 from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer
@@ -28,11 +29,13 @@ server.version = config.main.server_version
 application = service.Application('oonibackend')
 serviceCollection = service.IServiceCollection(application)
 
-internet.TCPServer(int(config.main.http_port),
+if config.main.http_port:
+    internet.TCPServer(int(config.main.http_port),
                    server.Site(HTTPBackend())
                   ).setServiceParent(serviceCollection)
 
-internet.SSLServer(int(config.main.ssl_port),
+if config.main.ssl_port:
+    internet.SSLServer(int(config.main.ssl_port),
                    server.Site(HTTPBackend()),
                    SSLContext(config),
                   ).setServiceParent(serviceCollection)
@@ -41,11 +44,24 @@ debugHTTPServer = DebugHTTPServer()
 internet.TCPServer(8090, debugHTTPServer).setServiceParent(serviceCollection)
 
 # Start the DNS Server related services
-TCPDNSServer = ProxyDNSServer()
-internet.TCPServer(int(config.main.dns_tcp_port), TCPDNSServer).setServiceParent(serviceCollection)
-UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer)
-internet.UDPServer(int(config.main.dns_udp_port), UDPFactory).setServiceParent(serviceCollection)
-
-# Start the ooni backend thing
-daphn3 = Daphn3Server()
-internet.TCPServer(int(config.main.daphn3_port), daphn3).setServiceParent(serviceCollection)
+if config.main.dns_tcp_port:
+    TCPDNSServer = ProxyDNSServer()
+    internet.TCPServer(int(config.main.dns_tcp_port),
+                       TCPDNSServer).setServiceParent(serviceCollection)
+
+if config.main.dns_udp_port:
+    UDPFactory = dns.DNSDatagramProtocol(TCPDNSServer)
+    internet.UDPServer(int(config.main.dns_udp_port),
+                       UDPFactory).setServiceParent(serviceCollection)
+
+# Start the OONI daphn3 backend
+if config.main.daphn3_port:
+    daphn3 = Daphn3Server()
+    internet.TCPServer(int(config.main.daphn3_port),
+                       daphn3).setServiceParent(serviceCollection)
+
+if config.main.reporting_port:
+    internet.TCPServer(int(config.main.reporting_port),
+                       reportingBackend).setServiceParent(serviceCollection)
+
+
diff --git a/oonib/report/__init__.py b/oonib/report/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/oonib/report/api.py b/oonib/report/api.py
index 04f70e0..ad0f62c 100644
--- a/oonib/report/api.py
+++ b/oonib/report/api.py
@@ -7,37 +7,61 @@ This is the async pcap reporting system. It requires the client to have created
 
 """
 import random
-import strings
-from twisted.internet import reactor
+import string
+from twisted.internet import reactor, defer
 from cyclone import web
+
+from oonib.report.db import models
 backend_version = '0.0.1'
 
 def generateReportID():
     size = 100
-    id = ''.join(random.choice(strings.ascii_letters) for x in range(size))
-    return id
+    report_id = ''.join(random.choice(string.ascii_letters) for x in range(size))
+    return report_id
 
+@xxxxxxxxxxxxxxxxxxxxx
 def newReport(software_name, software_version, test_name, test_version,
                progress, content):
+
+    report_id = generateReportID()
+
+    new_report = models.Report()
+
+    new_report.report_id = unicode(report_id)
+
+    new_report.software_name = unicode(software_name)
+    new_report.software_version = unicode(software_version)
+    new_report.test_name = unicode(test_name)
+    new_report.test_version = unicode(test_version)
+    new_report.progress = unicode(progress)
+    new_report.content = unicode(content)
+
     print "Software Name: %s" % software_name
     print "Software Version: %s" % software_version
     print "Test Name: %s" % test_name
     print "Test Version: %s" % test_version
     print "Progress: %s" % progress
     print "Content: %s" % content
-    reportId = generateReportID()
-    return {'backend_version': backend_version, 'report_id': reportID}
+
+    yield new_report.save()
+
+    defer.returnValue({'backend_version': backend_version, 'report_id':
+                        report_id})
 
 def updateReport(report_id, content):
     print "Report ID: %s" % report_id
     print "Content: %s" % content
-    return {'backend_version': '0.1', 'report_id': 'FOOBAR'}
 
-class DoReportHandler(web.RequestHandler):
+    return {'backend_version': backend_version, 'report_id': report_id}
+
+class NewReportHandler(web.RequestHandler):
     """
     Responsible for creating and updating reports.
     """
-    def post(self):
+
+    @web.asynchronous
+    @defer.inlineCallbacks
+    def get(self):
         """
         Creates a new report with the input
 
@@ -51,6 +75,10 @@ class DoReportHandler(web.RequestHandler):
            'content': 'XXX'
            }
 
+          Optional:
+            'test_helper': 'XXX'
+            'client_ip': 'XXX'
+
         * Response
 
           {'backend_version': 'XXX', 'report_id': 'XXX'}
@@ -61,15 +89,23 @@ class DoReportHandler(web.RequestHandler):
                      'test_name','test_version',
                      'progress', 'content']
         report = {}
-        error = None
         for arg in arguments:
             if len(self.get_arguments(arg)) == 0:
-                print "No %s specified" % arg
-                error = arg
-                break
+                raise web.HTTPError(400, "%s not specified as argument of POST"
+                        % arg)
             report[arg] = self.get_argument(arg)
-        if not error:
-            self.write(newReport(**report))
+
+        try:
+            test_helper = self.get_argument('test_helper')
+
+
+        except web.HTTPError:
+            pass
+
+        new_report = yield newReport(**report)
+
+        self.write(new_report)
+        self.finish()
 
     def put(self):
         """
@@ -88,11 +124,8 @@ class PCAPReportHandler(web.RequestHandler):
     def post(self):
         pass
 
-handlers = [(r"/report/do", DoReportHandler),
-            (r"/report/pcap", PCAPReportHandler)]
+spec = [(r"/report/new", NewReportHandler),
+        (r"/report/pcap", PCAPReportHandler)]
 
-reporting = web.Application(handlers)
-reactor.listenTCP(8888, reporting, interface="127.0.0.1")
-print "starting bullshit"
-reactor.run()
+reportingBackend = web.Application(spec)
 
diff --git a/oonib/report/db/__init__.py b/oonib/report/db/__init__.py
index 1248c19..178abe9 100644
--- a/oonib/report/db/__init__.py
+++ b/oonib/report/db/__init__.py
@@ -6,28 +6,28 @@ from twisted.internet.defer import inlineCallbacks
 from storm.twisted.transact import Transactor, transact
 from storm.locals import *
 
-dbtype = "sqlite"
+from storm.uri import URI
+from storm.databases.sqlite import SQLite
 
-if dbtype == "sqlite":
-    from storm.uri import URI
-    from storm.databases.sqlite import SQLite
-    database = SQLite(URI('sqlite:///test.db'))
+database = SQLite(URI('sqlite:///test.db'))
 
 threadpool = ThreadPool(0, 10)
 threadpool.start()
 transactor = Transactor(threadpool)
 
+def getStore():
+    store = Store(database)
+    return store
+
 @inlineCallbacks
 def create_tables():
-    def create(query):
-        store = Store(database)
-        store.execute(query)
-        store.commit()
+    from oonib.report.db import models
 
     for x in models.__all__:
-        query = getattr(models.__getattribute__(x), 'create_query')
+        query = getattr(models.__getattribute__(x), 'createQuery')
         try:
             yield transactor.run(create, query)
         except:
             log.msg("Failing in creating table for %s. Maybe it already exists?" % x)
 
+create_tables()
diff --git a/oonib/report/db/models.py b/oonib/report/db/models.py
index b4e56d0..21e60eb 100644
--- a/oonib/report/db/models.py
+++ b/oonib/report/db/models.py
@@ -1,7 +1,26 @@
+__all__ = ['Report', 'TestHelperTMP']
 from storm.twisted.transact import transact
 from storm.locals import *
 
-class Report(object):
+from oonib.report.db import getStore, transactor
+
+class OModel(object):
+
+    transactor = transactor
+
+    @transact
+    def create(query):
+        store = Store(database)
+        store.execute(query)
+        store.commit()
+
+    @transact
+    def save(self):
+        store = getStore()
+        store.add(self)
+        store.commit()
+
+class Report(OModel):
     """
     This represents an OONI Report as stored in the database.
 
@@ -27,6 +46,14 @@ class Report(object):
              than 100 we should append to the YAML data structure that is
              currently stored in such field.
     """
+    __storm_table__ = 'reports'
+
+    createQuery = "CREATE TABLE " + __storm_table__ +\
+                  "(id INTEGER PRIMARY KEY, report_id VARCHAR, software_name VARCHAR,"\
+                  "software_version VARCHAR, test_name VARCHAR, test_version VARCHAR,"\
+                  "progress VARCHAR, content VARCHAR)"
+
+
     id = Int(primary=True)
 
     report_id = Unicode()
@@ -39,3 +66,18 @@ class Report(object):
 
     content = Unicode()
 
+class TestHelperTMP(OModel):
+    __storm_table__ = 'testhelpertmp'
+
+    createQuery = "CREATE TABLE " + __storm_table__ +\
+                  "(id INTEGER PRIMARY KEY, report_id VARCHAR, test_helper VARCHAR,"\
+                  " client_ip VARCHAR, creation_time VARCHAR)"
+
+    id = Int(primary=True)
+
+    report_id = Unicode()
+
+    test_helper = Unicode()
+    client_ip = Unicode()
+
+    creation_time = Date()



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits