[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Start implementing support for downloading of test decks



commit fcae18e1cc5b6387bf4b609d62deec477414cd92
Author: Arturo Filastò <art@xxxxxxxxx>
Date:   Wed Aug 21 22:21:33 2013 +0200

    Start implementing support for downloading of test decks
    
    The work here was mainly merging together my branch with the one of @aagbsn and
    refactoring a lot of the code.
---
 ooni/deck.py        |   83 +++++++++++----------------------------------------
 ooni/nettest.py     |   27 ++++++++++++++++-
 ooni/oonibclient.py |   36 ++++++++++++++--------
 ooni/oonicli.py     |   14 ++++-----
 ooni/utils/net.py   |   19 ++++++++++++
 5 files changed, 93 insertions(+), 86 deletions(-)

diff --git a/ooni/deck.py b/ooni/deck.py
index 02c8fc1..f99916b 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -10,23 +10,6 @@ import os
 import re
 import yaml
 
-onionInputRegex =  re.compile("httpo://[a-z0-9]{16}\.onion/input/([a-z0-9]){40}$")
-
-@xxxxxxxxxxxxxxxxxxxxx
-def downloadFile(fileURL, filePath):
-
-    def writeFile(response_body, filePath): 
-        f = open(filePath, 'w+')
-        f.write(response_body)
-
-    finished = defer.Deferred()
-    finished.addCallback(writeFile, filePath)
-
-    agent = Agent(reactor, sockshost="127.0.0.1",
-            socksport=int(config.tor.socks_port))
-    response = yield agent.request("GET", fileURL)
-    response.deliverBody(BodyReceiver(finished))
-
 def verifyFile(filePath):
     # get the filename component of the file path
     digest = os.path.basename(filePath)
@@ -35,10 +18,11 @@ def verifyFile(filePath):
         return sha1digest.hexdigest() == digest
     return False
 
-class TestDeck(object):
-    def __init__(self, deckFile=None):
+class Deck(object):
+    def __init__(self, oonibclient, deckFile=None):
         self.netTestLoaders = []
         self.inputs = []
+
         if deckFile: self.loadDeck(deckFile)
 
     def loadDeck(self, deckFile):
@@ -63,54 +47,21 @@ class TestDeck(object):
     @defer.inlineCallbacks
     def fetchAndVerifyNetTestInput(self, net_test_loader):
         """ fetch and verify a single NetTest's inputs """
-        for test_class, test_methods in net_test_loader.testCases:
-            if test_class.inputFile:
-                inputArg = test_class.inputFile[0]
-                inputFileURL = test_class.localOptions[inputArg]
-    
-                m = onionInputRegex.match(inputFileURL)
-                if m:
-                    fileDigest = m.group(1)
-                else:
-                    fileDigest = os.path.basename(inputFileURL)
+        for input_file in net_test_loader.inputFiles:
+            if 'url' in input_file:
+                oonib = OONIBClient(input_file['address'])
 
-                cachedInputDir = os.path.join(config.advanced.data_dir,
+                cached_input_dir = os.path.join(config.advanced.data_dir,
                         'inputs')
-                cachedPath = os.path.join(cachedInputDir, fileDigest)
-                self.inputs.append(cachedPath)
+                cached_path = os.path.join(cached_input_dir, input_file['hash'])
+                self.inputs.append(cached_path)
 
-                if os.path.exists(cachedPath) and verifyFile(cachedPath):
-                        test_class.localOptions[inputArg] = cachedPath
-                        continue
-                if m:
-                    yield downloadFile(inputFileURL, cachedPath)
-                    if verifyFile(cachedPath):
-                        test_class.localOptions[inputArg] = cachedPath
+                if os.path.exists(cached_path) and verifyFile(cached_path):
+                        test_class.localOptions[inputArg] = cached_path
                         continue
-
-                raise UnableToLoadDeckInput, cachedPath
-
-def test_verify_file_success(): 
-    f = open('/dev/urandom')
-    r = f.read(1024*1024)
-    z = sha1(f).hexdigest()
-    f.close()
-    fn = '/tmp/%s' % z
-    f = open(fn)
-    f.write(r)
-    f.close()
-    verifyFile(fn)
-    os.unlink(fn)
-
-def test_verify_file_failure():
-    pass
-def test_load_deck_with_no_inputs(deck):
-    pass
-def test_load_deck_with_cached_input(deckFile):
-    pass
-def test_load_deck_with_evil_path(deckFile):
-    pass
-def test_load_deck_with_download_success(deckFile):
-    pass
-def test_load_deck_with_download_failure(deckFile):
-    pass
+                yield oonib.downloadInput(input_file['hash'], cached_path)
+                if verifyFile(cached_path):
+                    test_class.localOptions[input_file['key']] = cached_path
+                    continue
+                
+                raise UnableToLoadDeckInput, cached_path
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 51c6074..4806bdd 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -173,8 +173,9 @@ class NetTestLoader(object):
     method_prefix = 'test'
 
     def __init__(self, options, test_file=None, test_string=None):
+        self.onionInputRegex =  re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]){40}$")
         self.options = options
-        test_cases = None
+        self.testCases, test_cases = None, None
 
         if test_file:
             test_cases = loadNetTestFile(test_file)
@@ -183,6 +184,30 @@ class NetTestLoader(object):
 
         if test_cases:
             self.setupTestCases(test_cases)
+    
+    @property
+    def inputFiles(self):
+        input_files = []
+        if not self.testCases:
+            return input_files
+
+        for test_class, test_methods in self.testCases:
+            if test_class.inputFile:
+                key = test_class.inputFile[0]
+                filename = test_class.localOptions[key]
+                input_file = {
+                    'id': key
+                }
+                m = self.onionInputRegex.match(filename)
+                if m:
+                    input_file['url'] = filename
+                    input_file['address'] = m.group(1)
+                    input_file['hash'] = m.group(2)
+                else:
+                    input_file['filename'] = filename
+                input_files.append(input_file)
+
+        return input_files
 
     @property
     def testDetails(self):
diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py
index 28e42a7..3f9ed56 100644
--- a/ooni/oonibclient.py
+++ b/ooni/oonibclient.py
@@ -17,27 +17,21 @@ class InputFile(object):
 
         self._file = None
 
-class Deck(object):
-    pass
-
 class OONIBClient(object):
     def __init__(self, address):
         self.address = address
         self.agent = Agent(reactor)
 
-    def queryBackend(self, method, urn, query=None):
+    def _request(self, method, urn, genReceiver, bodyProducer=None):
         finished = defer.Deferred()
 
-        bodyProducer = None
-        if query:
-            bodyProducer = StringProducer(json.dumps(query))
-
         uri = self.address + urn
         d = self.agent.request(method, uri, bodyProducer)
+
         @d.addCallback
-        def cb(response):
+        def callback(response):
             content_length = response.headers.getRawHeaders('content-length')
-            response.deliverBody(BodyReceiver(finished, content_length, json.loads))
+            response.deliverBody(genReceiver(finished, content_length))
 
         @d.addErrback
         def eb(err):
@@ -45,6 +39,26 @@ class OONIBClient(object):
 
         return finished
 
+    def queryBackend(self, method, urn, query=None):
+        bodyProducer = None
+        if query:
+            bodyProducer = StringProducer(json.dumps(query), bodyProducer)
+    
+        def genReceiver(finished, content_length):
+            return BodyReceiver(finished, content_length, json.loads)
+
+        return self._request(method, urn, genReceiver, bodyProducer)
+
+    def download(self, urn, download_path):
+
+        def genReceiver(finished, content_length):
+            return Downloader(download_path, finished, content_length)
+
+        return self._request('GET', urn, genReceiver)
+    
+    def downloadInput(self, input_hash, download_path):
+        return self.download('/input/'+input_hash, download_path)
+
     def getNettestPolicy(self):
         pass
 
@@ -53,5 +67,3 @@ class OONIBClient(object):
 
     def getInputPolicy(self):
         pass
-
-
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 4c7dd8b..8708642 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -14,7 +14,7 @@ from ooni import errors
 
 from ooni.settings import config
 from ooni.director import Director
-from ooni.deck import TestDeck
+from ooni.deck import Deck
 from ooni.reporter import YAMLReporter, OONIBReporter
 from ooni.nettest import NetTestLoader, MissingRequiredOption
 
@@ -125,19 +125,19 @@ def runWithDirector():
     director = Director()
     d = director.start()
 
-    test_deck = TestDeck()
+    deck = Deck()
     if global_options['no-collector']:
         log.msg("Not reporting using a collector")
         collector = global_options['collector'] = None
 
     try:
         if global_options['testdeck']:
-            test_deck.loadDeck(global_options['testdeck'])
+            deck.loadDeck(global_options['testdeck'])
         else:
             log.debug("No test deck detected")
             net_test_loader = NetTestLoader(global_options['subargs'],
                     test_file=global_options['test_file'])
-            test_deck.insert(net_test_loader)
+            deck.insert(net_test_loader)
     except MissingRequiredOption, option_name:
         log.err('Missing required option: "%s"' % option_name)
         print net_test_loader.usageOptions().getUsage()
@@ -149,7 +149,7 @@ def runWithDirector():
 
     def fetch_nettest_inputs(result):
         try: 
-            test_deck.fetchAndVerifyDeckInputs()
+            deck.fetchAndVerifyDeckInputs()
         except errors.UnableToLoadDeckInput, e:
             return defer.failure.Failure(result)
 
@@ -174,8 +174,8 @@ def runWithDirector():
     # Wait until director has started up (including bootstrapping Tor)
     # before adding tests
     def post_director_start(_):
-        for net_test_loader in test_deck.netTestLoaders:
-            # TestDecks can specify different collectors
+        for net_test_loader in deck.netTestLoaders:
+            # Decks can specify different collectors
             # for each net test, so that each NetTest
             # may be paired with a test_helper and its collector
             # However, a user can override this behavior by
diff --git a/ooni/utils/net.py b/ooni/utils/net.py
index b63a173..db2ea18 100644
--- a/ooni/utils/net.py
+++ b/ooni/utils/net.py
@@ -81,6 +81,25 @@ class BodyReceiver(protocol.Protocol):
             self.data = self.body_processor(self.data)
         self.finished.callback(self.data)
 
+class Downloader(protocol.Protocol):
+    def __init__(self,  download_path,
+                 finished, content_length=None):
+        self.finished = finished
+        self.bytes_remaining = content_length
+        self.fp = open(download_path, 'w+')
+
+    def dataReceived(self, b):
+        self.fp.write(b)
+        if self.bytes_remaining:
+            if self.bytes_remaining == 0:
+                self.connectionLost(None)
+            else:
+                self.bytes_remaining -= len(b)
+
+    def connectionLost(self, reason):
+        self.fp.close()
+        self.finished.callback(self.download_path)
+
 def getSystemResolver():
     """
     XXX implement a function that returns the resolver that is currently



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits