[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [oonib/master] Remove dependency from Storm



commit fd52ee29a712f23b2e6c82922a27f848c0ffd98b
Author: Arturo Filastò <art@xxxxxxxxx>
Date:   Sat Jun 8 11:48:32 2013 +0200

    Remove dependency from Storm
    
    We are currently not using the db so there is no need for this.
---
 oonib/__init__.py    |   17 ++-----
 oonib/db/__init__.py |   30 ------------
 oonib/db/tables.py   |  123 -----------------------------------------------
 oonib/models.py      |  129 --------------------------------------------------
 oonib/oonibackend.py |    3 --
 oonib/report/api.py  |   55 +--------------------
 requirements.txt     |    2 -
 7 files changed, 4 insertions(+), 355 deletions(-)

diff --git a/oonib/__init__.py b/oonib/__init__.py
index 30e446a..c374f53 100644
--- a/oonib/__init__.py
+++ b/oonib/__init__.py
@@ -6,18 +6,14 @@
 In here we shall keep track of all variables and objects that should be
 instantiated only once and be common to pieces of GLBackend code.
 """
-__all__ = ['database', 'db_threadpool', 'Storage', 'randomStr']
 
-from twisted.python.threadpool import ThreadPool
+__version__ = '0.9.1'
+
+__all__ = ['Storage', 'randomStr']
 
-from storm.uri import URI
-from storm.twisted.transact import Transactor
-from storm.databases.sqlite import SQLite
 import string
 import random
 
-__version__ = '0.9.1'
-
 class Storage(dict):
     """
     A Storage object is like a dictionary except `obj.foo` can be used
@@ -69,10 +65,3 @@ def randomStr(length, num=True):
     if num:
         chars += string.digits
     return ''.join(random.choice(chars) for x in range(length))
-
-from oonib import config
-
-database = SQLite(URI(config.main.database_uri))
-db_threadpool = ThreadPool(0, config.main.db_threadpool_size)
-db_threadpool.start()
-transactor = Transactor(db_threadpool)
diff --git a/oonib/db/__init__.py b/oonib/db/__init__.py
deleted file mode 100644
index 2839747..0000000
--- a/oonib/db/__init__.py
+++ /dev/null
@@ -1,30 +0,0 @@
-__all__ = ['createTables', 'database', 'transactor']
-
-from twisted.python.threadpool import ThreadPool
-from twisted.internet.defer import inlineCallbacks, returnValue, Deferred
-
-from storm.locals import Store
-from storm.uri import URI
-from storm.databases.sqlite import SQLite
-
-from oonib import database, transactor
-from oonib import log
-
-@inlineCallbacks
-def createTables():
-    """
-    XXX this is to be refactored and only exists for experimentation.
-    """
-    from oonib.db import models
-    for model_name in models.__all__:
-        try:
-            model = getattr(m, model_name)
-        except Exception, e:
-            log.err("Error in db initting")
-            log.err(e)
-        try:
-            log.debug("Creating %s" % model)
-            yield tables.runCreateTable(model, transactor, database)
-        except Exception, e:
-            log.debug(str(e))
-
diff --git a/oonib/db/tables.py b/oonib/db/tables.py
deleted file mode 100644
index 908a295..0000000
--- a/oonib/db/tables.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# :authors: Arturo Filastò
-# :licence: see LICENSE
-
-from twisted.internet.defer import inlineCallbacks
-
-from storm.locals import Store
-from storm.properties import PropertyColumn
-from storm.exceptions import StormError
-
-from storm.variables import BoolVariable, DateTimeVariable, DateVariable
-from storm.variables import DecimalVariable, EnumVariable
-from storm.variables import FloatVariable, IntVariable, RawStrVariable
-from storm.variables import UnicodeVariable, JSONVariable, PickleVariable
-
-def variableToSQLite(var_type):
-    """
-    We take as input a storm.variable and we output the SQLite string it
-    represents.
-    """
-    sqlite_type = "VARCHAR"
-    if isinstance(var_type, BoolVariable):
-        sqlite_type = "INTEGER"
-    elif isinstance(var_type, DateTimeVariable):
-        pass
-        sqlite_type = ""
-    elif isinstance(var_type, DateVariable):
-        pass
-    elif isinstance(var_type, DecimalVariable):
-        pass
-    elif isinstance(var_type, EnumVariable):
-        sqlite_type = "BLOB"
-    elif isinstance(var_type, FloatVariable):
-        sqlite_type = "REAL"
-    elif isinstance(var_type, IntVariable):
-        sqlite_type = "INTEGER"
-    elif isinstance(var_type, RawStrVariable):
-        sqlite_type = "BLOB"
-    elif isinstance(var_type, UnicodeVariable):
-        pass
-    elif isinstance(var_type, JSONVariable):
-        sqlite_type = "BLOB"
-    elif isinstance(var_type, PickleVariable):
-        sqlite_type = "BLOB"
-    return "%s" % sqlite_type
-
-def varsToParametersSQLite(variables, primary_keys):
-    """
-    Takes as input a list of variables (convered to SQLite syntax and in the
-    form of strings) and primary_keys.
-    Outputs these variables converted into paramter syntax for SQLites.
-
-    ex.
-        variables: ["var1 INTEGER", "var2 BOOL", "var3 INTEGER"]
-        primary_keys: ["var1"]
-
-        output: "(var1 INTEGER, var2 BOOL, var3 INTEGER PRIMARY KEY (var1))"
-    """
-    params = "("
-    for var in variables[:-1]:
-        params += "%s %s, " % var
-    if len(primary_keys) > 0:
-        params += "%s %s, " % variables[-1]
-        params += "PRIMARY KEY ("
-        for key in primary_keys[:-1]:
-            params += "%s, " % key
-        params += "%s))" % primary_keys[-1]
-    else:
-        params += "%s %s)" % variables[-1]
-    return params
-
-def generateCreateQuery(model):
-    """
-    This takes as input a Storm model and outputs the creation query for it.
-    """
-    query = "CREATE TABLE "+ model.__storm_table__ + " "
-
-    variables = []
-    primary_keys = []
-
-    for attr in dir(model):
-        a = getattr(model, attr)
-        if isinstance(a, PropertyColumn):
-            var_stype = a.variable_factory()
-            var_type = variableToSQLite(var_stype)
-            name = a.name
-            variables.append((name, var_type))
-            if a.primary:
-                primary_keys.append(name)
-
-    query += varsToParametersSQLite(variables, primary_keys)
-    return query
-
-def createTable(model, transactor, database):
-    """
-    Create the table for the specified model.
-    Specification of a transactor and database is useful in unittesting.
-    """
-    if not transactor:
-        from oonib.db import transactor
-    if not database:
-        from oonib.db import database
-    store = Store(database)
-    create_query = generateCreateQuery(model)
-    try:
-        store.execute(create_query)
-    # XXX trap the specific error that is raised when the table exists
-    except StormError, e:
-        print "Failed to create table!"
-        print e
-        store.close()
-    store.commit()
-    store.close()
-
-@inlineCallbacks
-def runCreateTable(model, transactor=None, database=None):
-    """
-    Runs the table creation query wrapped in a transaction.
-    Transactions run in a separate thread.
-    """
-    yield transactor.run(createTable, model, transactor, database)
-
diff --git a/oonib/models.py b/oonib/models.py
deleted file mode 100644
index 60c0472..0000000
--- a/oonib/models.py
+++ /dev/null
@@ -1,129 +0,0 @@
-__all__ = ['Report', 'TestHelperTMP']
-from storm.twisted.transact import transact
-from storm.locals import *
-
-from oonib import randomStr
-from oonib import transactor
-
-def generateReportID():
-    """
-    Generates a report ID for usage in the database backed oonib collector.
-
-    XXX note how this function is different from the one in report/api.py
-    """
-    report_id = randomStr(100)
-    return report_id
-
-class OModel(object):
-
-    transactor = transactor
-
-    def getStore(self):
-        return Store(database)
-
-    @transact
-    def create(self, query):
-        store = Store(database)
-        store.execute(query)
-        store.commit()
-
-    @transact
-    def save(self):
-        store = getStore()
-        store.add(self)
-        store.commit()
-
-class Report(OModel):
-    """
-    This represents an OONI Report as stored in the database.
-
-    report_id: this is generated by the backend and is used by the client to
-               reference a previous report and append to it. It should be
-               treated as a shared secret between the probe and backend.
-
-    software_name: this indicates the name of the software performing the test
-                   (this will default to ooniprobe)
-
-    software_version: this is the version number of the software running the
-                      test.
-
-    test_name: the name of the test on which the report is being created.
-
-    test_version: indicates the version of the test
-
-    progress: what is the current progress of the report. This allows clients
-              to report event partial reports up to a certain percentage of
-              progress. Once the report is complete progress will be 100.
-
-    content: what is the content of the report. If the current progress is less
-             than 100 we should append to the YAML data structure that is
-             currently stored in such field.
-
-    XXX this is currently not used.
-    """
-    __storm_table__ = 'reports'
-
-    createQuery = "CREATE TABLE " + __storm_table__ +\
-                  "(id INTEGER PRIMARY KEY, report_id VARCHAR, software_name VARCHAR,"\
-                  "software_version VARCHAR, test_name VARCHAR, test_version VARCHAR,"\
-                  "progress VARCHAR, content VARCHAR)"
-
-
-    id = Int(primary=True)
-
-    report_id = Unicode()
-
-    software_name = Unicode()
-    software_version = Unicode()
-    test_name = Unicode()
-    test_version = Unicode()
-    progress = Int()
-
-    content = Unicode()
-
-    @transact
-    def new(report):
-        store = self.getStore()
-
-        print "Creating new report %s" % report
-
-        report_id = generateReportID()
-        new_report = models.Report()
-        new_report.report_id = unicode(report_id)
-
-        new_report.software_name = report['software_name']
-        new_report.software_version = report['software_version']
-        new_report.test_name = report['test_name']
-        new_report.test_version = report['test_version']
-        new_report.progress = report['progress']
-
-        if 'content' in report:
-            new_report.content = report['content']
-
-        print "Report: %s" % report
-
-        store.add(new_report)
-        try:
-            store.commit()
-        except:
-            store.close()
-
-        defer.returnValue({'backend_version': backend_version, 'report_id':
-                            report_id})
-
-
-class TestHelperTMP(OModel):
-    __storm_table__ = 'testhelpertmp'
-
-    createQuery = "CREATE TABLE " + __storm_table__ +\
-                  "(id INTEGER PRIMARY KEY, report_id VARCHAR, test_helper VARCHAR,"\
-                  " client_ip VARCHAR, creation_time VARCHAR)"
-
-    id = Int(primary=True)
-
-    report_id = Unicode()
-
-    test_helper = Unicode()
-    client_ip = Unicode()
-
-    creation_time = Date()
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index 0eef588..afd961f 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -20,7 +20,6 @@ from oonib.testhelpers import http_helpers, tcp_helpers
 
 from oonib import log
 
-from oonib import db_threadpool
 from oonib import config
 
 if config.main.uid and config.main.gid:
@@ -73,5 +72,3 @@ if config.helpers.http_return_request.port:
             int(config.helpers.http_return_request.port),
             http_helpers.HTTPReturnJSONHeadersHelper())
     http_return_request_helper.setServiceParent(serviceCollection)
-
-reactor.addSystemEventTrigger('after', 'shutdown', db_threadpool.stop)
diff --git a/oonib/report/api.py b/oonib/report/api.py
index d01f47c..7747a1e 100644
--- a/oonib/report/api.py
+++ b/oonib/report/api.py
@@ -20,7 +20,7 @@ from cyclone import web
 from oonib import otime
 from oonib import randomStr
 
-from oonib import models, config
+from oonib import config
 from oonib.report import file_collector
 
 def parseUpdateReportRequest(request):
@@ -45,59 +45,6 @@ def parseUpdateReportRequest(request):
 
     return parsed_request
 
-
-class NewReportHandlerDB(web.RequestHandler):
-    """
-    Responsible for creating and updating reports via database.
-    XXX this is not yet fully implemented.
-    """
-
-    @web.asynchronous
-    @defer.inlineCallbacks
-    def post(self):
-        """
-        Creates a new report with the input to the database.
-        XXX this is not yet implemented.
-
-        * Request
-
-          {'software_name': 'XXX',
-           'software_version': 'XXX',
-           'test_name': 'XXX',
-           'test_version': 'XXX',
-           'progress': 'XXX',
-           'content': 'XXX'
-           }
-
-          Optional:
-            'test_helper': 'XXX'
-            'client_ip': 'XXX'
-
-          * Response
-
-          {'backend_version': 'XXX', 'report_id': 'XXX'}
-
-        """
-        report_data = json.loads(self.request.body)
-        new_report = models.Report()
-        log.debug("Got this request %s" % report_data)
-        result = yield new_report.new(report_data)
-        self.write(result)
-        self.finish()
-
-    def put(self):
-        """
-        Update an already existing report with the database.
-
-        XXX this is not yet implemented.
-
-          {'report_id': 'XXX',
-           'content': 'XXX'
-          }
-        """
-        pass
-
-
 reportingBackendAPI = [
     (r"/report/([a-zA-Z0-9_\-]+)/close", file_collector.CloseReportHandlerFile),
     (r"/report", file_collector.NewReportHandlerFile),
diff --git a/requirements.txt b/requirements.txt
index b0d7f75..08fd114 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -32,8 +32,6 @@ pygeoip>=0.2.6
 ##
 https://people.torproject.org/~ioerror/src/mirrors/ooniprobe/scapy-02-25-2013-tip.zip#egg=scapy
 transaction>=1.4.1
-#https://pypi.python.org/packages/source/t/transaction/transaction-1.4.1.zip#md5=8db2680bc0f999219861a67b8f335a88#egg=transaction
-storm>=0.19
 #https://pypi.python.org/packages/source/s/storm/storm-0.19.tar.gz#md5=61d1ee4cd2a08639ab917e43fa2c9265#egg=storm
 txtorcon>=0.7
 pyOpenSSL>=0.13



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits