[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [stem/master] Move zstd and lzma checks to prereq module

To: tor-commits@xxxxxxxxxxxxxxxxxxxx
Subject: [tor-commits] [stem/master] Move zstd and lzma checks to prereq module
From: atagar@xxxxxxxxxxxxxx
Date: Tue, 8 May 2018 20:20:09 +0000 (UTC)
Delivered-to: archiver@xxxxxxxx
Delivery-date: Tue, 08 May 2018 16:20:26 -0400
List-archive: <http://lists.torproject.org/pipermail/tor-commits/>
List-help: <mailto:tor-commits-request@lists.torproject.org?subject=help>
List-id: "auto: code repository commits" <tor-commits.lists.torproject.org>
List-post: <mailto:tor-commits@lists.torproject.org>
List-subscribe: <https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=subscribe>
List-unsubscribe: <https://lists.torproject.org/cgi-bin/mailman/options/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=unsubscribe>
Patch-author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Sender: "tor-commits" <tor-commits-bounces@xxxxxxxxxxxxxxxxxxxx>

commit bb260c0ab9bd7d8fad08657b21a36a186e7a5fe0
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date:   Sat May 5 13:41:36 2018 -0700

    Move zstd and lzma checks to prereq module
    
    This is the spot where we do these prereq checks. Yay, remote module's finally
    under the 1k line threshold.
---
 stem/descriptor/remote.py      | 38 +++++--------------------------
 stem/prereq.py                 | 51 ++++++++++++++++++++++++++++++++++++++++--
 test/settings.cfg              |  1 +
 test/unit/descriptor/remote.py | 12 +++++-----
 4 files changed, 62 insertions(+), 40 deletions(-)

diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py
index 301e89d9..1b6a7ea6 100644
--- a/stem/descriptor/remote.py
+++ b/stem/descriptor/remote.py
@@ -113,29 +113,6 @@ try:
 except ImportError:
   import urllib2 as urllib
 
-try:
-  # added in python 3.3
-  import lzma
-  LZMA_SUPPORTED = True
-except ImportError:
-  LZMA_SUPPORTED = False
-
-try:
-  # We use the suggested python zstd library...
-  #
-  #   https://pypi.python.org/pypi/zstandard
-  #
-  # Unfortunately this installs as a zstd module which can be confused with...
-  #
-  #   https://pypi.python.org/pypi/zstd
-  #
-  # As such checking for the specific decompression class we'll need.
-
-  import zstd
-  ZSTD_SUPPORTED = hasattr(zstd, 'ZstdDecompressor')
-except ImportError:
-  ZSTD_SUPPORTED = False
-
 Compression = stem.util.enum.Enum(
   ('PLAINTEXT', 'identity'),
   ('GZIP', 'gzip'),  # can also be 'deflate'
@@ -143,9 +120,6 @@ Compression = stem.util.enum.Enum(
   ('LZMA', 'x-tor-lzma'),
 )
 
-ZSTD_UNAVAILABLE_MSG = 'ZSTD compression requires the zstandard module (https://pypi.python.org/pypi/zstandard)'
-LZMA_UNAVAILABLE_MSG = 'LZMA compression requires the lzma module (https://docs.python.org/3/library/lzma.html)'
-
 # Tor has a limited number of descriptors we can fetch explicitly by their
 # fingerprint or hashes due to a limit on the url length by squid proxies.
 
@@ -340,9 +314,10 @@ def _decompress(data, encoding):
   elif encoding in (Compression.GZIP, 'deflate'):
     return zlib.decompress(data, zlib.MAX_WBITS | 32).strip()
   elif encoding == Compression.ZSTD:
-    if not ZSTD_SUPPORTED:
+    if not stem.prereq.is_zstd_available():
       raise ImportError('Decompressing zstd data requires https://pypi.python.org/pypi/zstandard')
 
+    import zstd
     output_buffer = io.BytesIO()
 
     with zstd.ZstdDecompressor().write_to(output_buffer) as decompressor:
@@ -350,9 +325,10 @@ def _decompress(data, encoding):
 
     return output_buffer.getvalue().strip()
   elif encoding == Compression.LZMA:
-    if not LZMA_SUPPORTED:
+    if not stem.prereq.is_lzma_available():
       raise ImportError('Decompressing lzma data requires https://docs.python.org/3/library/lzma.html')
 
+    import lzma
     return lzma.decompress(data).strip()
   else:
     raise ValueError("'%s' isn't a recognized type of encoding" % encoding)
@@ -528,12 +504,10 @@ class Query(object):
       if isinstance(compression, str):
         compression = [compression]  # caller provided only a single option
 
-      if Compression.ZSTD in compression and not ZSTD_SUPPORTED:
-        log.log_once('stem.descriptor.remote.zstd_unavailable', log.INFO, ZSTD_UNAVAILABLE_MSG)
+      if Compression.ZSTD in compression and not stem.prereq.is_zstd_available():
         compression.remove(Compression.ZSTD)
 
-      if Compression.LZMA in compression and not LZMA_SUPPORTED:
-        log.log_once('stem.descriptor.remote.lzma_unavailable', log.INFO, LZMA_UNAVAILABLE_MSG)
+      if Compression.LZMA in compression and not stem.prereq.is_lzma_available():
         compression.remove(Compression.LZMA)
 
       if not compression:
diff --git a/stem/prereq.py b/stem/prereq.py
index 16437c23..6e230007 100644
--- a/stem/prereq.py
+++ b/stem/prereq.py
@@ -16,6 +16,8 @@ Checks for stem dependencies. We require python 2.6 or greater (including the
   is_python_3 - checks if python 3.0 or later is available
   is_sqlite_available - checks if the sqlite3 module is available
   is_crypto_available - checks if the cryptography module is available
+  is_zstd_available - checks if the zstd module is available
+  is_lzma_available - checks if the lzma module is available
   is_mock_available - checks if the mock module is available
 """
 
@@ -29,6 +31,8 @@ except ImportError:
   from stem.util.lru_cache import lru_cache
 
 CRYPTO_UNAVAILABLE = "Unable to import the cryptography module. Because of this we'll be unable to verify descriptor signature integrity. You can get cryptography from: https://pypi.python.org/pypi/cryptography";
+ZSTD_UNAVAILABLE = 'ZSTD compression requires the zstandard module (https://pypi.python.org/pypi/zstandard)'
+LZMA_UNAVAILABLE = 'LZMA compression requires the lzma module (https://docs.python.org/3/library/lzma.html)'
 PYNACL_UNAVAILABLE = "Unable to import the pynacl module. Because of this we'll be unable to verify descriptor ed25519 certificate integrity. You can get pynacl from https://pypi.python.org/pypi/PyNaCl/";
 
 
@@ -113,8 +117,6 @@ def is_crypto_available():
     otherwise
   """
 
-  from stem.util import log
-
   try:
     from cryptography.utils import int_from_bytes, int_to_bytes
     from cryptography.hazmat.backends import default_backend
@@ -127,11 +129,56 @@ def is_crypto_available():
 
     return True
   except ImportError:
+    from stem.util import log
     log.log_once('stem.prereq.is_crypto_available', log.INFO, CRYPTO_UNAVAILABLE)
     return False
 
 
 @lru_cache()
+def is_zstd_available():
+  """
+  Checks if the `zstd module <https://pypi.python.org/pypi/zstandard>`_ is
+  available.
+
+  .. versionadded:: 1.7.0
+
+  :returns: **True** if we can use the zstd module and **False** otherwise
+  """
+
+  try:
+    # Unfortunately the zstandard module uses the same namespace as another
+    # zstd module (https://pypi.python.org/pypi/zstd), so we need to
+    # differentiate them.
+
+    import zstd
+    return hasattr(zstd, 'ZstdDecompressor')
+  except ImportError:
+    from stem.util import log
+    log.log_once('stem.prereq.is_zstd_available', log.INFO, ZSTD_UNAVAILABLE)
+    return False
+
+
+@lru_cache()
+def is_lzma_available():
+  """
+  Checks if the `lzma module <https://docs.python.org/3/library/lzma.html>`_ is
+  available. This was added as a builtin in Python 3.3.
+
+  .. versionadded:: 1.7.0
+
+  :returns: **True** if we can use the lzma module and **False** otherwise
+  """
+
+  try:
+    import lzma
+    return True
+  except ImportError:
+    from stem.util import log
+    log.log_once('stem.prereq.is_lzma_available', log.INFO, LZMA_UNAVAILABLE)
+    return False
+
+
+@lru_cache()
 def is_mock_available():
   """
   Checks if the mock module is available. In python 3.3 and up it is a builtin
diff --git a/test/settings.cfg b/test/settings.cfg
index f1cbd381..60224463 100644
--- a/test/settings.cfg
+++ b/test/settings.cfg
@@ -171,6 +171,7 @@ pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.serialization.
 pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.ciphers.modes' imported but unused
 pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.ciphers.Cipher' imported but unused
 pyflakes.ignore stem/prereq.py => 'cryptography.hazmat.primitives.ciphers.algorithms' imported but unused
+pyflakes.ignore stem/prereq.py => 'lzma' imported but unused
 pyflakes.ignore stem/prereq.py => 'nacl.encoding' imported but unused
 pyflakes.ignore stem/prereq.py => 'nacl.signing' imported but unused
 pyflakes.ignore stem/interpreter/__init__.py => undefined name 'raw_input'
diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py
index a02a1171..fd831e1e 100644
--- a/test/unit/descriptor/remote.py
+++ b/test/unit/descriptor/remote.py
@@ -207,20 +207,20 @@ class TestDescriptorDownloader(unittest.TestCase):
     self.assertEqual(TEST_RESOURCE, query.resource)
 
   def test_zstd_support_check(self):
-    with patch('stem.descriptor.remote.ZSTD_SUPPORTED', True):
+    with patch('stem.prereq.is_zstd_available', Mock(return_value = True)):
       query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.ZSTD, start = False)
       self.assertEqual([Compression.ZSTD], query.compression)
 
-    with patch('stem.descriptor.remote.ZSTD_SUPPORTED', False):
+    with patch('stem.prereq.is_zstd_available', Mock(return_value = False)):
       query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.ZSTD, start = False)
       self.assertEqual([Compression.PLAINTEXT], query.compression)
 
   def test_lzma_support_check(self):
-    with patch('stem.descriptor.remote.LZMA_SUPPORTED', True):
+    with patch('stem.prereq.is_lzma_available', Mock(return_value = True)):
       query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.LZMA, start = False)
       self.assertEqual([Compression.LZMA], query.compression)
 
-    with patch('stem.descriptor.remote.LZMA_SUPPORTED', False):
+    with patch('stem.prereq.is_lzma_available', Mock(return_value = False)):
       query = stem.descriptor.remote.Query(TEST_RESOURCE, compression = Compression.LZMA, start = False)
       self.assertEqual([Compression.PLAINTEXT], query.compression)
 
@@ -260,7 +260,7 @@ class TestDescriptorDownloader(unittest.TestCase):
     Download a zstd compressed descriptor.
     """
 
-    if not stem.descriptor.remote.ZSTD_SUPPORTED:
+    if not stem.prereq.is_zstd_available():
       self.skipTest('(requires zstd module)')
       return
 
@@ -279,7 +279,7 @@ class TestDescriptorDownloader(unittest.TestCase):
     Download a lzma compressed descriptor.
     """
 
-    if not stem.descriptor.remote.LZMA_SUPPORTED:
+    if not stem.prereq.is_lzma_available():
       self.skipTest('(requires lzma module)')
       return
 



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits

Prev by Author: [tor-commits] [stem/master] Fix python3 proc regression
Next by Author: [tor-commits] [stem/master] Example of v3 hidden services in create_ephemeral_hidden_service docs
Previous by thread: [tor-commits] [tor/master] Avoid unsigned integer underflow on empty input.
Next by thread: [tor-commits] [stem/master] Replace get_authorities() with DirectoryAuthority.from_cache()
Index(es):
- Author
- Thread