[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Refactor verification of input file and implement caching



commit fb8a90c6f2ec136eb477bbb2bf91d61456e66c80
Author: Arturo Filastò <art@xxxxxxxxx>
Date:   Thu Aug 22 17:38:38 2013 +0200

    Refactor verification of input file and implement caching
---
 ooni/deck.py                   |   28 +++---------
 ooni/oonibclient.py            |   93 ++++++++++++++++++++++++++++++++--------
 ooni/tests/test_oonibclient.py |    2 +-
 3 files changed, 83 insertions(+), 40 deletions(-)

diff --git a/ooni/deck.py b/ooni/deck.py
index f99916b..ef1b58a 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -1,6 +1,5 @@
 #-*- coding: utf-8 -*-
 
-from hashlib import sha1
 from ooni.nettest import NetTestLoader
 from ooni.settings import config
 from ooni.utils.txagentwithsocks import Agent
@@ -10,14 +9,6 @@ import os
 import re
 import yaml
 
-def verifyFile(filePath):
-    # get the filename component of the file path
-    digest = os.path.basename(filePath)
-    with open(filePath) as f:
-        sha1digest = sha1(f.read())
-        return sha1digest.hexdigest() == digest
-    return False
-
 class Deck(object):
     def __init__(self, oonibclient, deckFile=None):
         self.netTestLoaders = []
@@ -51,17 +42,10 @@ class Deck(object):
             if 'url' in input_file:
                 oonib = OONIBClient(input_file['address'])
 
-                cached_input_dir = os.path.join(config.advanced.data_dir,
-                        'inputs')
-                cached_path = os.path.join(cached_input_dir, input_file['hash'])
-                self.inputs.append(cached_path)
-
-                if os.path.exists(cached_path) and verifyFile(cached_path):
-                        test_class.localOptions[inputArg] = cached_path
-                        continue
-                yield oonib.downloadInput(input_file['hash'], cached_path)
-                if verifyFile(cached_path):
-                    test_class.localOptions[input_file['key']] = cached_path
-                    continue
+                input_file = yield oonib.downloadInput(input_file['hash'])
+                try:
+                    input_file.verify()
+                except AssertionError:
+                    raise UnableToLoadDeckInput, cached_path
                 
-                raise UnableToLoadDeckInput, cached_path
+                test_class.localOptions[input_file['key']] = input_file.cached_file
diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py
index 4f555b2..9665ee1 100644
--- a/ooni/oonibclient.py
+++ b/ooni/oonibclient.py
@@ -1,30 +1,67 @@
-from hashlib import sha256
-
+import os
 import json
 
+from hashlib import sha256
+
 from twisted.internet import defer, reactor
 from twisted.web.client import Agent
 
+from ooni.settings import config
 from ooni.utils import log
 from ooni.utils.net import BodyReceiver, StringProducer, Downloader
 
 class InputFile(object):
-    def __init__(self, descriptor):
-        self.id = descriptor['id']
+    def __init__(self, input_hash):
+        self.id = input_hash
+        cached_input_dir = os.path.join(config.advanced.data_dir,
+                'inputs')
+        cache_path = os.path.join(cached_input_dir, input_hash)
+        self.cached_file = cache_path
+        self.cached_descriptor = cache_path + '.desc'
+    
+    @property
+    def descriptorCached(self):
+        if os.path.exists(self.cached_descriptor):
+            with open(self.cached_descriptor) as f:
+                descriptor = json.load(f)
+                self.load(descriptor)
+            return True
+        return False
+    
+    @property
+    def fileCached(self):
+        if os.path.exists(self.cached_file):
+            try:
+                self.verify()
+            except AssertionError:
+                log.err("The input %s failed validation. Going to consider it not cached." % self.id)
+                return False
+            return True
+        return False
+
+    def save(self):
+        with open(self.cached_descriptor, 'w+') as f:
+            json.dump({
+                'name': self.name,
+                'id': self.id,
+                'version': self.version,
+                'author': self.author,
+                'date': self.date,
+                'description': self.description
+            }, f)
+
+    def load(self, descriptor):
         self.name = descriptor['name']
         self.version = descriptor['version']
         self.author = descriptor['author']
         self.date = descriptor['date']
         self.description = descriptor['description']
 
-        self.file_path = None
-
     def verify(self):
-        digest = os.path.basename(self.file_path)
-        with open(self.file_path) as f:
+        digest = os.path.basename(self.cached_file)
+        with open(self.cached_file) as f:
             file_hash = sha256(f.read())
-            return file_hash.hexdigest() == digest
-        return False
+            assert file_hash.hexdigest() == digest
 
 class OONIBClient(object):
     def __init__(self, address):
@@ -73,25 +110,47 @@ class OONIBClient(object):
         pass
 
     def getInput(self, input_hash):
-        try:
-            return defer.succeed(self.input_files[input_hash])
-        except KeyError:
+        input_file = InputFile(input_hash)
+        if input_file.descriptorCached:
+            return defer.succeed(input_file)
+        else:
             d = self.queryBackend('GET', '/input/' + input_hash)
+
             @d.addCallback
             def cb(descriptor):
-                self.input_files[input_hash] = InputFile(descriptor)
-                return self.input_files[input_hash]
+                input_file.load(descriptor)
+                input_file.save()
+                return input_file
+
             @d.addErrback
             def err(err):
                 log.err("Failed to get descriptor for input %s" % input_hash)
                 log.exception(err)
+
             return d
 
     def getInputList(self):
         return self.queryBackend('GET', '/input')
 
-    def downloadInput(self, input_hash, download_path):
-        return self.download('/input/'+input_hash+'/file', download_path)
+    def downloadInput(self, input_hash):
+        input_file = InputFile(input_hash)
+
+        if input_file.fileCached:
+            return defer.succeed(input_file)
+        else:
+            d = self.download('/input/'+input_hash+'/file', input_file.cached_file)
+
+            @d.addCallback
+            def cb(res):
+                input_file.verify()
+                return input_file
+
+            @d.addErrback
+            def err(err):
+                log.err("Failed to download the input file %s" % input_hash)
+                log.exception(err)
+
+            return d
 
     def getInputPolicy(self):
         pass
diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py
index 02f9f8f..26e2f62 100644
--- a/ooni/tests/test_oonibclient.py
+++ b/ooni/tests/test_oonibclient.py
@@ -28,7 +28,7 @@ class TestOONIBClient(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_download_input(self):
-        yield self.oonibclient.downloadInput(input_id, input_id)
+        yield self.oonibclient.downloadInput(input_id)
 
     @defer.inlineCallbacks
     def test_get_deck_list(self):



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits