[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Add support for looking up decks and deck descriptors in oonibclient



commit 0b62769ebed74e62882b06ee4579e78799df3d64
Author: Arturo Filastò <art@xxxxxxxxx>
Date:   Tue Aug 27 17:09:05 2013 +0200

    Add support for looking up decks and deck descriptors in oonibclient
---
 ooni/deck.py        |   70 ++++++++++++++++++++++++++++++++++--
 ooni/oonibclient.py |   99 +++++++++++++++++++++++----------------------------
 2 files changed, 112 insertions(+), 57 deletions(-)

diff --git a/ooni/deck.py b/ooni/deck.py
index 23f4cb1..3b0716c 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -5,20 +5,82 @@ from ooni.settings import config
 from ooni.utils import log
 from ooni.utils.txagentwithsocks import Agent
 from ooni import errors as e
-from ooni.oonibclient import OONIBClient
 
 from twisted.internet import reactor, defer
 
 import os
 import re
 import yaml
+import json
+from hashlib import sha256
 
-class Deck(object):
-    def __init__(self, deckFile=None):
+class InputFile(object):
+    def __init__(self, input_hash):
+        self.id = input_hash
+        cached_input_dir = os.path.join(config.advanced.data_dir,
+                'inputs')
+        cache_path = os.path.join(cached_input_dir, input_hash)
+        self.cached_file = cache_path
+        self.cached_descriptor = cache_path + '.desc'
+    
+    @property
+    def descriptorCached(self):
+        if os.path.exists(self.cached_descriptor):
+            with open(self.cached_descriptor) as f:
+                descriptor = json.load(f)
+                self.load(descriptor)
+            return True
+        return False
+    
+    @property
+    def fileCached(self):
+        if os.path.exists(self.cached_file):
+            try:
+                self.verify()
+            except AssertionError:
+                log.err("The input %s failed validation. Going to consider it not cached." % self.id)
+                return False
+            return True
+        return False
+
+    def save(self):
+        with open(self.cached_descriptor, 'w+') as f:
+            json.dump({
+                'name': self.name,
+                'id': self.id,
+                'version': self.version,
+                'author': self.author,
+                'date': self.date,
+                'description': self.description
+            }, f)
+    
+    def load(self, descriptor):
+        self.name = descriptor['name']
+        self.version = descriptor['version']
+        self.author = descriptor['author']
+        self.date = descriptor['date']
+        self.description = descriptor['description']
+
+    def verify(self):
+        digest = os.path.basename(self.cached_file)
+        with open(self.cached_file) as f:
+            file_hash = sha256(f.read())
+            assert file_hash.hexdigest() == digest
+
+class Deck(InputFile):
+    def __init__(self, deck_hash=None, deckFile=None):
+        self.id = deck_hash
         self.bouncer = None
         self.netTestLoaders = []
         self.inputs = []
         self.testHelpers = {}
+
+        cached_deck_dir = os.path.join(config.advanced.data_dir,
+                'decks')
+        cache_path = os.path.join(cached_deck_dir, deck_hash)
+        self.cached_file = cache_path
+        self.cached_descriptor = cache_path + '.desc'
+ 
         if deckFile: self.loadDeck(deckFile)
 
     def loadDeck(self, deckFile):
@@ -49,6 +111,7 @@ class Deck(object):
 
     @defer.inlineCallbacks
     def lookupTestHelpers(self):
+        from ooni.oonibclient import OONIBClient
         oonibclient = OONIBClient(self.bouncer)
         required_test_helpers = []
         for net_test_loader in self.netTestLoaders:
@@ -73,6 +136,7 @@ class Deck(object):
     @defer.inlineCallbacks
     def fetchAndVerifyNetTestInput(self, net_test_loader):
         """ fetch and verify a single NetTest's inputs """
+        from ooni.oonibclient import OONIBClient
         log.debug("Fetching and verifying inputs")
         for i in net_test_loader.inputFiles:
             if 'url' in i:
diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py
index 9dada5f..24dc374 100644
--- a/ooni/oonibclient.py
+++ b/ooni/oonibclient.py
@@ -7,64 +7,12 @@ from twisted.internet import defer, reactor
 
 from ooni.utils.txagentwithsocks import Agent
 
+from ooni.deck import Deck, InputFile
 from ooni import errors as e
 from ooni.settings import config
 from ooni.utils import log
 from ooni.utils.net import BodyReceiver, StringProducer, Downloader
 
-class InputFile(object):
-    def __init__(self, input_hash):
-        self.id = input_hash
-        cached_input_dir = os.path.join(config.advanced.data_dir,
-                'inputs')
-        cache_path = os.path.join(cached_input_dir, input_hash)
-        self.cached_file = cache_path
-        self.cached_descriptor = cache_path + '.desc'
-    
-    @property
-    def descriptorCached(self):
-        if os.path.exists(self.cached_descriptor):
-            with open(self.cached_descriptor) as f:
-                descriptor = json.load(f)
-                self.load(descriptor)
-            return True
-        return False
-    
-    @property
-    def fileCached(self):
-        if os.path.exists(self.cached_file):
-            try:
-                self.verify()
-            except AssertionError:
-                log.err("The input %s failed validation. Going to consider it not cached." % self.id)
-                return False
-            return True
-        return False
-
-    def save(self):
-        with open(self.cached_descriptor, 'w+') as f:
-            json.dump({
-                'name': self.name,
-                'id': self.id,
-                'version': self.version,
-                'author': self.author,
-                'date': self.date,
-                'description': self.description
-            }, f)
-    
-    def load(self, descriptor):
-        self.name = descriptor['name']
-        self.version = descriptor['version']
-        self.author = descriptor['author']
-        self.date = descriptor['date']
-        self.description = descriptor['description']
-
-    def verify(self):
-        digest = os.path.basename(self.cached_file)
-        with open(self.cached_file) as f:
-            file_hash = sha256(f.read())
-            assert file_hash.hexdigest() == digest
-
 class Collector(object):
     def __init__(self, address):
         self.address = address
@@ -193,6 +141,50 @@ class OONIBClient(object):
     def getNettestPolicy(self):
         return self.queryBackend('GET', '/policy/nettest')
 
+    def getDeckList(self):
+        return self.queryBackend('GET', '/deck')
+
+    def getDeck(self, deck_hash):
+        deck = Deck(deck_hash)
+        if deck.descriptorCached:
+            return defer.succeed(deck)
+        else:
+            d = self.queryBackend('GET', '/deck/' + deck_hash)
+
+            @d.addCallback
+            def cb(descriptor):
+                deck.load(descriptor)
+                deck.save()
+                return deck
+
+            @d.addErrback
+            def err(err):
+                log.err("Failed to get descriptor for deck %s" % deck_hash)
+                print err
+                log.exception(err)
+
+            return d
+
+    def downloadDeck(self, deck_hash):
+        deck = Deck(deck_hash)
+        if deck.fileCached:
+            return defer.succeed(deck)
+        else:
+            d = self.download('/deck/'+deck_hash+'/file', deck.cached_file)
+
+            @d.addCallback
+            def cb(res):
+                deck.verify()
+                return deck
+
+            @d.addErrback
+            def err(err):
+                log.err("Failed to download the deck %s" % deck_hash)
+                print err
+                log.exception(err)
+
+            return d
+
     @defer.inlineCallbacks
     def lookupTestCollector(self, test_name):
         try:
@@ -216,4 +208,3 @@ class OONIBClient(object):
             raise e.CouldNotFindTestHelper
 
         defer.returnValue(test_helper)
-



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits