[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Cleanup some of the log outputs.



commit cf333afdf5dc08e17d02516a0af79552aa6f0275
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Thu Jul 28 01:38:41 2016 +0200

    Cleanup some of the log outputs.
    
    * Fix more unittests.
    
    Will you be green green green?
---
 ooni/agent/scheduler.py      |  2 +-
 ooni/director.py             |  6 +++---
 ooni/reporter.py             |  7 +++++--
 ooni/resources.py            | 30 ++++++++++++++----------------
 ooni/tests/test_geoip.py     | 16 +++++++++++++++-
 ooni/tests/test_oonicli.py   |  5 ++---
 ooni/tests/test_resources.py | 16 +++++++++++++++-
 ooni/tests/test_utils.py     |  8 ++++----
 ooni/utils/onion.py          |  1 -
 9 files changed, 59 insertions(+), 32 deletions(-)

diff --git a/ooni/agent/scheduler.py b/ooni/agent/scheduler.py
index 9784f8a..71a7edb 100644
--- a/ooni/agent/scheduler.py
+++ b/ooni/agent/scheduler.py
@@ -97,7 +97,7 @@ def run_system_tasks(no_input_store=False):
 
     if no_input_store:
         log.debug("Not updating the inputs")
-        task_classes.pop(UpdateInputsAndResources)
+        task_classes.remove(UpdateInputsAndResources)
 
     for task_class in task_classes:
         task = task_class()
diff --git a/ooni/director.py b/ooni/director.py
index ad6d1e2..f6311ac 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -141,7 +141,7 @@ class Director(object):
         self._tor_starting.addCallback(self._tor_startup_success)
 
     def _tor_startup_failure(self, failure):
-        log.msg("Failed to start tor")
+        log.err("Failed to start tor")
         log.exception(failure)
         self._reset_tor_state()
         self.notify(DirectorEvent("error",
@@ -382,7 +382,7 @@ class Director(object):
             yield self._tor_starting
             defer.returnValue(self._tor_state)
 
-        log.msg("Starting Tor...")
+        log.msg("Starting Tor")
         self._tor_state = 'starting'
         if check_incoherences:
             try:
@@ -405,7 +405,7 @@ class Director(object):
                 data_dir = os.path.expanduser(config.tor.data_dir)
 
                 if not os.path.exists(data_dir):
-                    log.msg("%s does not exist. Creating it." % data_dir)
+                    log.debug("%s does not exist. Creating it." % data_dir)
                     os.makedirs(data_dir)
                 tor_config.DataDirectory = data_dir
 
diff --git a/ooni/reporter.py b/ooni/reporter.py
index e2f155f..20a13f5 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -342,9 +342,12 @@ class OONIBReporter(OReporter):
             log.msg("Try running a different test or try reporting to a "
                     "different collector.")
             raise errors.OONIBReportCreationError
-        except Exception, e:
+        except errors.OONIBError:
             log.err("Failed to connect to reporter backend")
-            log.exception(e)
+            raise errors.OONIBReportCreationError
+        except Exception as exc:
+            log.err("Failed to connect to reporter backend")
+            log.exception(exc)
             raise errors.OONIBReportCreationError
 
         self.reportId = response['report_id'].encode('ascii')
diff --git a/ooni/resources.py b/ooni/resources.py
index 47ebf86..aef0f13 100644
--- a/ooni/resources.py
+++ b/ooni/resources.py
@@ -1,24 +1,25 @@
-import gzip
 import json
-import shutil
 
 from twisted.python.filepath import FilePath
 from twisted.internet import defer
 from twisted.web.client import downloadPage, getPage, HTTPClientFactory
 
+from ooni.utils import log, gunzip, rename
+from ooni.settings import config
+
 # Disable logs of HTTPClientFactory
 HTTPClientFactory.noisy = False
 
-from ooni.utils import log, gunzip, rename
-from ooni.settings import config
 
 class UpdateFailure(Exception):
     pass
 
+
 def get_download_url(tag_name, filename):
     return ("https://github.com/OpenObservatory/ooni-resources/releases";
             "/download/{0}/{1}".format(tag_name, filename))
 
+
 def get_current_version():
     manifest = FilePath(config.resources_directory).child("manifest.json")
     if not manifest.exists():
@@ -27,6 +28,7 @@ def get_current_version():
         manifest = json.load(f)
     return int(manifest["version"])
 
+
 @defer.inlineCallbacks
 def get_latest_version():
     """
@@ -41,7 +43,8 @@ def get_latest_version():
 
 
 def get_out_of_date_resources(current_manifest, new_manifest,
-                              country_code=None):
+                              country_code=None,
+                              resources_directory=config.resources_directory):
     current_res = {}
     new_res = {}
     for r in current_manifest["resources"]:
@@ -51,11 +54,13 @@ def get_out_of_date_resources(current_manifest, new_manifest,
         new_res[r["path"]] = r
 
     paths_to_delete = [
-        current_res[path] for path in list(set(current_res.keys()) -
-                                           set(new_res.keys()))
-        ]
+        current_res[path] for path in list(
+                set(current_res.keys()) -
+                set(new_res.keys())
+            )
+    ]
     paths_to_update = []
-    _resources = FilePath(config.resources_directory)
+    _resources = FilePath(resources_directory)
     for path, info in new_res.items():
         if (country_code is not None and
                 info["country_code"] != "ALL" and
@@ -78,13 +83,6 @@ def get_out_of_date_resources(current_manifest, new_manifest,
 
     return paths_to_update, paths_to_delete
 
-def gunzip(file_path):
-    tmp_location = FilePath(file_path).temporarySibling()
-    in_file = gzip.open(file_path)
-    with tmp_location.open('w') as out_file:
-        shutil.copyfileobj(in_file, out_file)
-    in_file.close()
-    rename(tmp_location.path, file_path)
 
 @defer.inlineCallbacks
 def check_for_update(country_code=None):
diff --git a/ooni/tests/test_geoip.py b/ooni/tests/test_geoip.py
index 8eb964d..b5e76f3 100644
--- a/ooni/tests/test_geoip.py
+++ b/ooni/tests/test_geoip.py
@@ -1,4 +1,5 @@
-
+import os
+import shutil
 from twisted.internet import defer
 
 from ooni.tests import is_internet_connected, bases
@@ -23,6 +24,17 @@ class TestGeoIP(bases.ConfigTestCase):
         assert len(res.split('.')) == 4
 
     def test_geoip_database_version(self):
+        maxmind_dir = os.path.join(self.config.resources_directory,
+                                   'maxmind-geoip')
+        try:
+            os.mkdir(maxmind_dir)
+        except OSError:
+            pass
+        with open(os.path.join(maxmind_dir, 'GeoIP.dat'), 'w+') as f:
+            f.write("XXX")
+        with open(os.path.join(maxmind_dir, 'GeoIPASNum.dat'), 'w+') as f:
+            f.write("XXX")
+
         version = geoip.database_version()
         assert 'GeoIP' in version.keys()
         assert 'GeoIPASNum' in version.keys()
@@ -31,3 +43,5 @@ class TestGeoIP(bases.ConfigTestCase):
         assert isinstance(version['GeoIP']['timestamp'], float)
         assert len(version['GeoIPASNum']['sha256']) == 64
         assert isinstance(version['GeoIPASNum']['timestamp'], float)
+
+        shutil.rmtree(maxmind_dir)
diff --git a/ooni/tests/test_oonicli.py b/ooni/tests/test_oonicli.py
index c3facee..06a21ff 100644
--- a/ooni/tests/test_oonicli.py
+++ b/ooni/tests/test_oonicli.py
@@ -55,7 +55,6 @@ advanced:
     oonid_api_port: 8042
 tor:
     socks_port: 9050
-
 """ % config.data_directory
 
 
@@ -149,8 +148,8 @@ class TestRunDirector(ConfigTestCase):
 
         yield self.run_helper('blocking/dns_consistency',
                               ['-b', '8.8.8.8:53',
-                               '-t', '8.8.8.8',
-                               '-f', 'example-input.txt'],
+                              '-t', '8.8.8.8',
+                              '-f', 'example-input.txt'],
                               verify_function)
 
     @defer.inlineCallbacks
diff --git a/ooni/tests/test_resources.py b/ooni/tests/test_resources.py
index 45473e9..6d1bb3c 100644
--- a/ooni/tests/test_resources.py
+++ b/ooni/tests/test_resources.py
@@ -1,3 +1,7 @@
+import os
+import shutil
+import tempfile
+
 from ooni.resources import get_out_of_date_resources, check_for_update
 from ooni.tests.bases import ConfigTestCase
 
@@ -36,7 +40,17 @@ class TestResourceUpdate(ConfigTestCase):
         return check_for_update()
 
     def test_resources_out_of_date(self):
+        tmp_dir = tempfile.mkdtemp()
+        os.mkdir(os.path.join(tmp_dir, 'some'))
+        original_paths = map(lambda r: r['path'],
+                             SAMPLE_CURRENT_MANIFEST['resources'])
+        for path in original_paths:
+            with open(os.path.join(tmp_dir, path), 'w+'):
+                pass
         paths_to_update, paths_to_delete = get_out_of_date_resources(
-            SAMPLE_CURRENT_MANIFEST, SAMPLE_NEW_MANIFEST)
+            SAMPLE_CURRENT_MANIFEST, SAMPLE_NEW_MANIFEST,
+            resources_directory=tmp_dir)
         self.assertEqual(paths_to_update[0]["path"], "some/file-to-update.txt")
         self.assertEqual(paths_to_delete[0]["path"], "some/file-to-delete.txt")
+
+        shutil.rmtree(tmp_dir)
diff --git a/ooni/tests/test_utils.py b/ooni/tests/test_utils.py
index bbaa26b..ea1d858 100644
--- a/ooni/tests/test_utils.py
+++ b/ooni/tests/test_utils.py
@@ -26,19 +26,19 @@ class TestUtils(unittest.TestCase):
 
     def test_generate_filename(self):
         filename = generate_filename(self.test_details)
-        self.assertEqual(filename, 'foo-2016-01-01T012222Z')
+        self.assertEqual(filename, '20160101T012222Z-ZZ-AS0-foo')
 
     def test_generate_filename_with_extension(self):
         filename = generate_filename(self.test_details, extension=self.extension)
-        self.assertEqual(filename, 'foo-2016-01-01T012222Z.ext')
+        self.assertEqual(filename, '20160101T012222Z-ZZ-AS0-foo.ext')
 
     def test_generate_filename_with_prefix(self):
         filename = generate_filename(self.test_details, prefix=self.prefix)
-        self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z')
+        self.assertEqual(filename, 'prefix-20160101T012222Z-ZZ-AS0-foo')
 
     def test_generate_filename_with_extension_and_prefix(self):
         filename = generate_filename(self.test_details, prefix=self.prefix, extension=self.extension)
-        self.assertEqual(filename, 'prefix-foo-2016-01-01T012222Z.ext')
+        self.assertEqual(filename, 'prefix-20160101T012222Z-ZZ-AS0-foo.ext')
 
     def test_get_addresses(self):
         addresses = net.getAddresses()
diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py
index e18a6ee..df9dfec 100644
--- a/ooni/utils/onion.py
+++ b/ooni/utils/onion.py
@@ -244,7 +244,6 @@ class TorLauncherWithRetries(object):
     @defer.inlineCallbacks
     def _state_complete(self, state):
         config.tor_state = state
-        log.msg("Successfully bootstrapped Tor")
         log.debug("We now have the following circuits: ")
         for circuit in state.circuits.values():
             log.debug(" * %s" % circuit)



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits