[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [stem/master] Download method for bandwidth files
commit 16ac148a98b19f6c9649ab1f548bd3c5410428fc
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date: Thu Aug 15 14:51:30 2019 -0700
Download method for bandwidth files
---
stem/descriptor/collector.py | 53 ++++++++++++++++++---
test/unit/descriptor/collector.py | 19 +++++++-
.../data/collector/bandwidths-2019-05-cropped.tar | Bin 0 -> 32768 bytes
3 files changed, 63 insertions(+), 9 deletions(-)
diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py
index 6a4c700f..e8e798b3 100644
--- a/stem/descriptor/collector.py
+++ b/stem/descriptor/collector.py
@@ -53,8 +53,10 @@ With this you can either download and read directly from CollecTor...
|- get_extrainfo_descriptors - published extrainfo descriptors
|- get_microdescriptors - published microdescriptors
|- get_consensus - published router status entries
+ |
|- get_key_certificates - authority key certificates
- +- get_exit_list - TorDNSEL exit list
+ |- get_bandwidth_files - bandwidth authority heuristics
+ +- get_exit_lists - TorDNSEL exit list
File - Individual file residing within CollecTor
|- read - provides descriptors from this file
@@ -65,8 +67,10 @@ With this you can either download and read directly from CollecTor...
|- get_extrainfo_descriptors - published extrainfo descriptors
|- get_microdescriptors - published microdescriptors
|- get_consensus - published router status entries
+ |
|- get_key_certificates - authority key certificates
- |- get_exit_list - TorDNSEL exit list
+ |- get_bandwidth_files - bandwidth authority heuristics
+ |- get_exit_lists - TorDNSEL exit list
|
|- index - metadata for content available from CollecTor
+- files - files available from CollecTor
@@ -205,14 +209,25 @@ def get_key_certificates(start = None, end = None, cache_to = None, timeout = No
yield desc
-def get_exit_list(start = None, end = None, cache_to = None, timeout = None, retries = 3):
+def get_bandwidth_files(start = None, end = None, cache_to = None, timeout = None, retries = 3):
+ """
+ Shorthand for
+ :func:`~stem.descriptor.collector.CollecTor.get_bandwidth_files`
+ on our singleton instance.
+ """
+
+ for desc in get_instance().get_bandwidth_files(start, end, cache_to, timeout, retries):
+ yield desc
+
+
+def get_exit_lists(start = None, end = None, cache_to = None, timeout = None, retries = 3):
"""
Shorthand for
- :func:`~stem.descriptor.collector.CollecTor.get_exit_list`
+ :func:`~stem.descriptor.collector.CollecTor.get_exit_lists`
on our singleton instance.
"""
- for desc in get_instance().get_exit_list(start, end, cache_to, timeout, retries):
+ for desc in get_instance().get_exit_lists(start, end, cache_to, timeout, retries):
yield desc
@@ -594,9 +609,33 @@ class CollecTor(object):
for desc in f.read(cache_to, 'dir-key-certificate-3', timeout = timeout, retries = retries):
yield desc
- def get_exit_list(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
+ def get_bandwidth_files(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
+ """
+ Bandwidth authority heuristics for the given time range, sorted oldest to
+ newest.
+
+ :param datetime.datetime start: time range to begin with
+ :param datetime.datetime end: time range to end with
+ :param str cache_to: directory to cache archives into, if an archive is
+ available here it is not downloaded
+ :param int timeout: timeout for downloading each individual archive when
+ the connection becomes idle, no timeout applied if **None**
+ :param int retries: maximum attempts to impose on a per-archive basis
+
+ :returns: **iterator** of
+ :class:`~stem.descriptor.bandwidth_file.BandwidthFile
+ for the given time range
+
+ :raises: :class:`~stem.DownloadFailed` if the download fails
+ """
+
+ for f in self.files('bandwidth-file', start, end):
+ for desc in f.read(cache_to, 'bandwidth-file', timeout = timeout, retries = retries):
+ yield desc
+
+ def get_exit_lists(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
"""
- `TorDNSEL exit list <https://www.torproject.org/projects/tordnsel.html.en>`_
+ `TorDNSEL exit lists <https://www.torproject.org/projects/tordnsel.html.en>`_
for the given time range, sorted oldest to newest.
:param datetime.datetime start: time range to begin with
diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py
index e3ba1f40..44893fab 100644
--- a/test/unit/descriptor/collector.py
+++ b/test/unit/descriptor/collector.py
@@ -344,13 +344,28 @@ class TestCollector(unittest.TestCase):
@patch('stem.util.connection.download')
@patch('stem.descriptor.collector.CollecTor.files')
- def test_reading_exit_list(self, files_mock, download_mock):
+ def test_reading_bandwidth_files(self, files_mock, download_mock):
+ with open(get_resource('collector/bandwidths-2019-05-cropped.tar'), 'rb') as archive:
+ download_mock.return_value = archive.read()
+
+ files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/bandwidths/bandwidths-2019-05.tar', 12345, '2016-09-04 09:21')]
+
+ descriptors = list(stem.descriptor.collector.get_bandwidth_files())
+ self.assertEqual(2, len(descriptors))
+
+ f = descriptors[0]
+ self.assertEqual('BandwidthFile', type(f).__name__)
+ self.assertEqual(22, len(f.measurements))
+
+ @patch('stem.util.connection.download')
+ @patch('stem.descriptor.collector.CollecTor.files')
+ def test_reading_exit_lists(self, files_mock, download_mock):
with open(get_resource('collector/exit-list-2018-11-cropped.tar'), 'rb') as archive:
download_mock.return_value = archive.read()
files_mock.return_value = [stem.descriptor.collector.File('archive/exit-lists/exit-list-2018-11.tar', 12345, '2016-09-04 09:21')]
- descriptors = list(stem.descriptor.collector.get_exit_list())
+ descriptors = list(stem.descriptor.collector.get_exit_lists())
self.assertEqual(3713, len(descriptors))
f = descriptors[0]
diff --git a/test/unit/descriptor/data/collector/bandwidths-2019-05-cropped.tar b/test/unit/descriptor/data/collector/bandwidths-2019-05-cropped.tar
new file mode 100644
index 00000000..4f3c8724
Binary files /dev/null and b/test/unit/descriptor/data/collector/bandwidths-2019-05-cropped.tar differ
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits