[tor-commits] [stem/master] Initial download function

[atagar@xxxxxxxxxxxxxx]

commit 4196e37f0f43d033b23f740c1e3e3d9612ba4655
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date:   Mon Jul 29 19:49:37 2019 -0700

    Initial download function
    
    Few rough edges, but a copy-paste of the initial pydoc demo now works.
---
 stem/descriptor/collector.py       | 69 +++++++++++++++++++++++++++++++++++---
 stem/descriptor/remote.py          |  6 +---
 test/integ/descriptor/collector.py |  4 +--
 3 files changed, 67 insertions(+), 12 deletions(-)

diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py
index f76fa225..a78d60c4 100644
--- a/stem/descriptor/collector.py
+++ b/stem/descriptor/collector.py
@@ -16,7 +16,7 @@ With this you can either download and read directly from CollecTor...
   import datetime
   import stem.descriptor.collector
 
-  yesterday = datetime.date.today() - datetime.timedelta(1)
+  yesterday = datetime.datetime.today() - datetime.timedelta(1)
 
   # provide yesterday's exits
 
@@ -33,7 +33,7 @@ With this you can either download and read directly from CollecTor...
   import stem.descriptor
   import stem.descriptor.collector
 
-  yesterday = datetime.date.today() - datetime.timedelta(1)
+  yesterday = datetime.datetime.today() - datetime.timedelta(1)
   path = os.path.expanduser('~/descriptor_cache/server_desc_today')
 
   with open(path, 'wb') as cache_file:
@@ -53,6 +53,7 @@ import datetime
 import json
 import os
 import re
+import shutil
 import sys
 import tempfile
 import time
@@ -70,6 +71,7 @@ except ImportError:
 
 COLLECTOR_URL = 'https://collector.torproject.org/'
 REFRESH_INDEX_RATE = 3600  # get new index if cached copy is an hour old
+SINGLETON_COLLECTOR = None
 
 YEAR_DATE = re.compile('-(\\d{4})-(\\d{2})\\.')
 SEC_DATE = re.compile('(\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2})')
@@ -113,6 +115,52 @@ COLLECTOR_DESC_TYPES = {
 }
 
 
+def get_instance():
+  """
+  Provides the singleton :class:`~stem.descriptor.collector.CollecTor`
+  used for this module's shorthand functions.
+
+  :returns: singleton :class:`~stem.descriptor.collector.CollecTor` instance
+  """
+
+  global SINGLETON_COLLECTOR
+
+  if SINGLETON_COLLECTOR is None:
+    SINGLETON_COLLECTOR = CollecTor()
+
+  return SINGLETON_COLLECTOR
+
+
+def get_server_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3):
+  """
+  Provides server descriptors for the given time range, sorted oldest to
+  newest.
+
+  :param datetime.datetime start: time range to begin with
+  :param datetime.datetime end: time range to end with
+  :param str cache_to: directory to cache archives into, if an archive is
+    available here it is not downloaded
+  :param int timeout: timeout for downloading each individual archive when the
+    connection becomes idle, no timeout applied if **None**
+  :param int retires: maximum attempts to impose on a per-archive basis
+
+  :returns: **iterator** of
+    :class:`~stem.descriptor.server_descriptor.ServerDescriptor` for the given
+    time range
+
+  :raises:
+    * **socket.timeout** if our request timed out
+    * **urllib2.URLError** for most request failures
+
+    Note that the urllib2 module may fail with other exception types, in
+    which case we'll pass it along.
+  """
+
+  for f in get_instance().files('server-descriptor', start, end):
+    for desc in f.read(cache_to, timeout = timeout, retries = retries):
+      yield desc
+
+
 def _download(url, timeout, retries):
   """
   Download from the given url.
@@ -229,13 +277,24 @@ class File(object):
       if self._downloaded_to and os.path.exists(self._downloaded_to):
         directory = os.path.dirname(self._downloaded_to)
       else:
-        with tempfile.TemporaryDirectory() as tmp_directory:
-          return self.read(tmp_directory, descriptor_type, timeout, retries)
+        # TODO: The following can be replaced with simpler usage of
+        # tempfile.TemporaryDirectory when we drop python 2.x support.
+
+        tmp_directory = tempfile.mkdtemp()
+
+        for desc in self.read(tmp_directory, descriptor_type, timeout, retries):
+          yield desc
+
+        shutil.rmtree(tmp_directory)
+
+        return
 
     # TODO: the following will not work if the tar contains multiple types or a type we do not support
 
     path = self.download(directory, True, timeout, retries)
-    return parse_file(path, descriptor_type)
+
+    for desc in parse_file(path, descriptor_type):
+      yield desc
 
   def download(self, directory, decompress = True, timeout = None, retries = 3):
     """
diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py
index 251fc26f..af24f624 100644
--- a/stem/descriptor/remote.py
+++ b/stem/descriptor/remote.py
@@ -164,11 +164,7 @@ DIR_PORT_BLACKLIST = ('tor26', 'Serge')
 def get_instance():
   """
   Provides the singleton :class:`~stem.descriptor.remote.DescriptorDownloader`
-  used for the following functions...
-
-    * :func:`stem.descriptor.remote.get_server_descriptors`
-    * :func:`stem.descriptor.remote.get_extrainfo_descriptors`
-    * :func:`stem.descriptor.remote.get_consensus`
+  used for this module's shorthand functions.
 
   .. versionadded:: 1.5.0
 
diff --git a/test/integ/descriptor/collector.py b/test/integ/descriptor/collector.py
index dbb09d5a..6a0ec5ac 100644
--- a/test/integ/descriptor/collector.py
+++ b/test/integ/descriptor/collector.py
@@ -35,8 +35,8 @@ class TestCollector(unittest.TestCase):
     if compression and not compression.available:
       self.skipTest('(%s unavailable)' % compression)
 
-    collector = CollecTor(compression = compression)
-    index = collector.index()
+    collector = CollecTor()
+    index = collector.index(compression = compression)
 
     self.assertEqual('https://collector.torproject.org', index['path'])
     self.assertEqual(['archive', 'contrib', 'recent'], [entry['path'] for entry in index['directories']])



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits