[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [stem/master] Drop url helper function



commit a60efda6275fbee76349896ba6f1ed63415c4b94
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date:   Fri Jul 5 14:12:52 2019 -0700

    Drop url helper function
    
    Originally I was unsure if I'd want this. Turns out nope - it helped with url
    testability, but we can exercise the same with better mocks.
---
 stem/descriptor/collector.py      |  31 +++---------
 test/unit/descriptor/collector.py | 102 +++++++++++++++++++++++++++++++-------
 2 files changed, 89 insertions(+), 44 deletions(-)

diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py
index 746e4b1d..d723e97a 100644
--- a/stem/descriptor/collector.py
+++ b/stem/descriptor/collector.py
@@ -70,34 +70,11 @@ COLLECTOR_URL = 'https://collector.torproject.org/'
 REFRESH_INDEX_RATE = 3600  # get new index if cached copy is an hour old
 
 
-def url(resource, compression = Compression.PLAINTEXT):
-  """
-  Provides CollecTor url for the given resource.
-
-  :param str resource: resource type of the url
-  :param descriptor.Compression compression: compression type to
-    download from
-
-  :returns: **str** with the CollecTor url
-  """
-
-  # TODO: Unsure how to most elegantly map resources to urls. No doubt
-  # this'll change as we add more types.
-
-  if resource == 'index':
-    path = ('index', 'index.json')
-  else:
-    raise ValueError("'%s' isn't a recognized resource type" % resource)
-
-  extension = compression.extension if compression not in (None, Compression.PLAINTEXT) else ''
-  return COLLECTOR_URL + '/'.join(path) + extension
-
-
 def _download(url, compression, timeout, retries):
   """
   Download from the given url.
 
-  :param str url: url to download from
+  :param str url: uncompressed url to download from
   :param descriptor.Compression compression: decompression type
   :param int timeout: timeout when connection becomes idle, no timeout applied
     if **None**
@@ -115,6 +92,10 @@ def _download(url, compression, timeout, retries):
   """
 
   start_time = time.time()
+  extension = compression.extension if compression not in (None, Compression.PLAINTEXT) else ''
+
+  if not url.endswith(extension):
+    url += extension
 
   try:
     response = urllib.urlopen(url, timeout = timeout).read()
@@ -185,7 +166,7 @@ class CollecTor(object):
     """
 
     if not self._cached_index or time.time() - self._cached_index_at >= REFRESH_INDEX_RATE:
-      response = _download(url('index', self.compression), self.compression, self.timeout, self.retries)
+      response = _download(COLLECTOR_URL + 'index/index.json', self.compression, self.timeout, self.retries)
       self._cached_index = json.loads(response)
       self._cached_index_at = time.time()
 
diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py
index c46fb60c..cfb57c3d 100644
--- a/test/unit/descriptor/collector.py
+++ b/test/unit/descriptor/collector.py
@@ -8,7 +8,7 @@ import unittest
 import stem.prereq
 
 from stem.descriptor import Compression
-from stem.descriptor.collector import CollecTor, url
+from stem.descriptor.collector import CollecTor
 
 try:
   # added in python 3.3
@@ -18,34 +18,95 @@ except ImportError:
 
 URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen'
 
+MINIMAL_INDEX = {
+  'index_created': '2017-12-25 21:06',
+  'build_revision': '56a303e',
+  'path': 'https://collector.torproject.org'
+}
+
+MINIMAL_INDEX_JSON = b'{"index_created":"2017-12-25 21:06","build_revision":"56a303e","path":"https://collector.torproject.org"}'
+
 
 class TestCollector(unittest.TestCase):
-  def test_url(self):
-    self.assertEqual('https://collector.torproject.org/index/index.json', url('index'))
-    self.assertEqual('https://collector.torproject.org/index/index.json', url('index', compression = None))
-    self.assertEqual('https://collector.torproject.org/index/index.json', url('index', compression = Compression.PLAINTEXT))
-    self.assertEqual('https://collector.torproject.org/index/index.json.gz', url('index', compression = Compression.GZIP))
-    self.assertEqual('https://collector.torproject.org/index/index.json.bz2', url('index', compression = Compression.BZ2))
-    self.assertEqual('https://collector.torproject.org/index/index.json.xz', url('index', compression = Compression.LZMA))
+  @patch(URL_OPEN)
+  def test_download_plaintext(self, urlopen_mock):
+    urlopen_mock.return_value = io.BytesIO(MINIMAL_INDEX_JSON)
+
+    collector = CollecTor(compression = Compression.PLAINTEXT)
+    self.assertEqual(MINIMAL_INDEX, collector.index())
+    urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json', timeout = None)
 
   @patch(URL_OPEN)
-  def test_retries(self, urlopen_mock):
-    collector = CollecTor(retries = 4)
+  def test_download_gzip(self, urlopen_mock):
+    if not Compression.GZIP.available:
+      self.skipTest('(gzip compression unavailable)')
+      return
+
+    import zlib
+    urlopen_mock.return_value = io.BytesIO(zlib.compress(MINIMAL_INDEX_JSON))
+
+    collector = CollecTor(compression = Compression.GZIP)
+    self.assertEqual(MINIMAL_INDEX, collector.index())
+    urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.gz', timeout = None)
+
+  @patch(URL_OPEN)
+  def test_download_bz2(self, urlopen_mock):
+    if not Compression.BZ2.available:
+      self.skipTest('(bz2 compression unavailable)')
+      return
+
+    import bz2
+    urlopen_mock.return_value = io.BytesIO(bz2.compress(MINIMAL_INDEX_JSON))
+
+    collector = CollecTor(compression = Compression.BZ2)
+    self.assertEqual(MINIMAL_INDEX, collector.index())
+    urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.bz2', timeout = None)
+
+  @patch(URL_OPEN)
+  def test_download_lzma(self, urlopen_mock):
+    if not Compression.LZMA.available:
+      self.skipTest('(lzma compression unavailable)')
+      return
+
+    import lzma
+    urlopen_mock.return_value = io.BytesIO(lzma.compress(MINIMAL_INDEX_JSON))
+
+    collector = CollecTor(compression = Compression.LZMA)
+    self.assertEqual(MINIMAL_INDEX, collector.index())
+    urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.lzma', timeout = None)
+
+  @patch(URL_OPEN)
+  def test_download_zstd(self, urlopen_mock):
+    if not Compression.ZSTD.available:
+      self.skipTest('(zstd compression unavailable)')
+      return
+
+    import zstd
+    compressor = zstd.ZstdCompressor()
+    urlopen_mock.return_value = io.BytesIO(compressor.compress(MINIMAL_INDEX_JSON))
+
+    collector = CollecTor(compression = Compression.ZSTD)
+    self.assertEqual(MINIMAL_INDEX, collector.index())
+    urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.zst', timeout = None)
+
+  @patch(URL_OPEN)
+  def test_download_retries(self, urlopen_mock):
     urlopen_mock.side_effect = IOError('boom')
 
+    collector = CollecTor(retries = 0)
+    self.assertRaisesRegexp(IOError, 'boom', collector.index)
+    self.assertEqual(1, urlopen_mock.call_count)
+
+    urlopen_mock.reset_mock()
+
+    collector = CollecTor(retries = 4)
     self.assertRaisesRegexp(IOError, 'boom', collector.index)
     self.assertEqual(5, urlopen_mock.call_count)
 
-  @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'{"index_created":"2017-12-25 21:06","build_revision":"56a303e","path":"https://collector.torproject.org"}')))
+  @patch(URL_OPEN, Mock(return_value = io.BytesIO(MINIMAL_INDEX_JSON)))
   def test_index(self):
-    expected = {
-      'index_created': '2017-12-25 21:06',
-      'build_revision': '56a303e',
-      'path': 'https://collector.torproject.org'
-    }
-
     collector = CollecTor(compression = Compression.PLAINTEXT)
-    self.assertEqual(expected, collector.index())
+    self.assertEqual(MINIMAL_INDEX, collector.index())
 
   @patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not json')))
   def test_index_malformed_json(self):
@@ -57,7 +118,10 @@ class TestCollector(unittest.TestCase):
       self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', collector.index)
 
   def test_index_malformed_compression(self):
-    for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA):
+    for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA, Compression.ZSTD):
+      if not compression.available:
+        next
+
       with patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not compressed'))):
         collector = CollecTor(compression = compression)
         self.assertRaisesRegexp(IOError, 'Unable to decompress %s response' % compression, collector.index)



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits