[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [stem/master] Bridge descriptor support



commit e14d74473ff7319298a920129db6037079ab3652
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date:   Thu Aug 15 13:53:35 2019 -0700

    Bridge descriptor support
    
    Adding a 'bridge' argument to descriptor download methods to retrieve bridge
    counterparts instead.
---
 stem/descriptor/__init__.py                        |   2 +
 stem/descriptor/collector.py                       |  43 +++++++++--------
 stem/descriptor/networkstatus.py                   |   7 ++-
 stem/descriptor/router_status_entry.py             |  13 ++++++
 test/unit/descriptor/collector.py                  |  52 ++++++++++++++++++++-
 .../bridge-extra-infos-2019-03-cropped.tar         | Bin 0 -> 15872 bytes
 .../bridge-server-descriptors-2019-02-cropped.tar  | Bin 0 -> 9216 bytes
 .../collector/bridge-statuses-2019-05-cropped.tar  | Bin 0 -> 467456 bytes
 8 files changed, 93 insertions(+), 24 deletions(-)

diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index 2044c7bf..c099ca86 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -920,6 +920,8 @@ class Descriptor(object):
     :returns: :class:`~stem.descriptor.TypeAnnotation` with our type information
     """
 
+    # TODO: populate this from the archive instead if available (so we have correct version numbers)
+
     if self.TYPE_ANNOTATION_NAME is not None:
       return TypeAnnotation(self.TYPE_ANNOTATION_NAME, 1, 0)
     else:
diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py
index 99d02242..b7982426 100644
--- a/stem/descriptor/collector.py
+++ b/stem/descriptor/collector.py
@@ -146,25 +146,25 @@ def get_instance():
   return SINGLETON_COLLECTOR
 
 
-def get_server_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3):
+def get_server_descriptors(start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3):
   """
   Shorthand for
   :func:`~stem.descriptor.collector.CollecTor.get_server_descriptors`
   on our singleton instance.
   """
 
-  for desc in get_instance().get_server_descriptors(start, end, cache_to, timeout, retries):
+  for desc in get_instance().get_server_descriptors(start, end, cache_to, bridge, timeout, retries):
     yield desc
 
 
-def get_extrainfo_descriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3):
+def get_extrainfo_descriptors(start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3):
   """
   Shorthand for
   :func:`~stem.descriptor.collector.CollecTor.get_extrainfo_descriptors`
   on our singleton instance.
   """
 
-  for desc in get_instance().get_extrainfo_descriptors(start, end, cache_to, timeout, retries):
+  for desc in get_instance().get_extrainfo_descriptors(start, end, cache_to, bridge, timeout, retries):
     yield desc
 
 
@@ -179,14 +179,14 @@ def get_microdescriptors(start = None, end = None, cache_to = None, timeout = No
     yield desc
 
 
-def get_consensus(start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, timeout = None, retries = 3):
+def get_consensus(start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, bridge = False, timeout = None, retries = 3):
   """
   Shorthand for
   :func:`~stem.descriptor.collector.CollecTor.get_consensus`
   on our singleton instance.
   """
 
-  for desc in get_instance().get_consensus(start, end, cache_to, document_handler, version, microdescriptor, timeout, retries):
+  for desc in get_instance().get_consensus(start, end, cache_to, document_handler, version, microdescriptor, bridge, timeout, retries):
     yield desc
 
 
@@ -413,7 +413,7 @@ class CollecTor(object):
     self._cached_files = None
     self._cached_index_at = 0
 
-  def get_server_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
+  def get_server_descriptors(self, start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3):
     """
     Provides server descriptors published during the given time range, sorted
     oldest to newest.
@@ -422,6 +422,7 @@ class CollecTor(object):
     :param datetime.datetime end: time range to end with
     :param str cache_to: directory to cache archives into, if an archive is
       available here it is not downloaded
+    :param bool bridge: standard descriptors if **False**, bridge if **True**
     :param int timeout: timeout for downloading each individual archive when
       the connection becomes idle, no timeout applied if **None**
     :param int retires: maximum attempts to impose on a per-archive basis
@@ -433,13 +434,13 @@ class CollecTor(object):
     :raises: :class:`~stem.DownloadFailed` if the download fails
     """
 
-    # TODO: support bridge variants ('bridge-server-descriptor' type)
+    desc_type = 'server-descriptor' if not bridge else 'bridge-server-descriptor'
 
-    for f in self.files('server-descriptor', start, end):
-      for desc in f.read(cache_to, 'server-descriptor', timeout = timeout, retries = retries):
+    for f in self.files(desc_type, start, end):
+      for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries):
         yield desc
 
-  def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
+  def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, bridge = False, timeout = None, retries = 3):
     """
     Provides extrainfo descriptors published during the given time range,
     sorted oldest to newest.
@@ -448,6 +449,7 @@ class CollecTor(object):
     :param datetime.datetime end: time range to end with
     :param str cache_to: directory to cache archives into, if an archive is
       available here it is not downloaded
+    :param bool bridge: standard descriptors if **False**, bridge if **True**
     :param int timeout: timeout for downloading each individual archive when
       the connection becomes idle, no timeout applied if **None**
     :param int retires: maximum attempts to impose on a per-archive basis
@@ -459,10 +461,10 @@ class CollecTor(object):
     :raises: :class:`~stem.DownloadFailed` if the download fails
     """
 
-    # TODO: support bridge variants ('bridge-extra-info' type)
+    desc_type = 'extra-info' if not bridge else 'bridge-extra-info'
 
-    for f in self.files('extra-info', start, end):
-      for desc in f.read(cache_to, 'extra-info', timeout = timeout, retries = retries):
+    for f in self.files(desc_type, start, end):
+      for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries):
         yield desc
 
   def get_microdescriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
@@ -498,7 +500,7 @@ class CollecTor(object):
       for desc in f.read(cache_to, 'microdescriptor', timeout = timeout, retries = retries):
         yield desc
 
-  def get_consensus(self, start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, timeout = None, retries = 3):
+  def get_consensus(self, start = None, end = None, cache_to = None, document_handler = DocumentHandler.ENTRIES, version = 3, microdescriptor = False, bridge = False, timeout = None, retries = 3):
     """
     Provides consensus router status entries published during the given time
     range, sorted oldest to newest.
@@ -512,6 +514,7 @@ class CollecTor(object):
     :param int version: consensus variant to retrieve (versions 2 or 3)
     :param bool microdescriptor: provides the microdescriptor consensus if
       **True**, standard consensus otherwise
+    :param bool bridge: standard descriptors if **False**, bridge if **True**
     :param int timeout: timeout for downloading each individual archive when
       the connection becomes idle, no timeout applied if **None**
     :param int retires: maximum attempts to impose on a per-archive basis
@@ -523,20 +526,20 @@ class CollecTor(object):
     :raises: :class:`~stem.DownloadFailed` if the download fails
     """
 
-    if version == 3 and not microdescriptor:
+    if version == 3 and not microdescriptor and not bridge:
       desc_type = 'network-status-consensus-3'
-    elif version == 3 and microdescriptor:
+    elif version == 3 and microdescriptor and not bridge:
       desc_type = 'network-status-microdesc-consensus-3'
-    elif version == 2 and not microdescriptor:
+    elif version == 2 and not microdescriptor and not bridge:
       desc_type = 'network-status-2'
+    elif bridge:
+      desc_type = 'bridge-network-status'
     else:
       if microdescriptor and version != 3:
         raise ValueError('Only v3 microdescriptors are available (not version %s)' % version)
       else:
         raise ValueError('Only v2 and v3 router status entries are available (not version %s)' % version)
 
-    # TODO: support bridge variants ('bridge-network-status' type)
-
     for f in self.files(desc_type, start, end):
       for desc in f.read(cache_to, desc_type, document_handler, timeout = timeout, retries = retries):
         yield desc
diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py
index b0589f2a..5f542c2f 100644
--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -93,6 +93,7 @@ from stem.descriptor import (
 
 from stem.descriptor.router_status_entry import (
   RouterStatusEntryV2,
+  RouterStatusEntryBridgeV2,
   RouterStatusEntryV3,
   RouterStatusEntryMicroV3,
 )
@@ -322,7 +323,7 @@ def _parse_file(document_file, document_type = None, validate = False, is_microd
   elif document_type == NetworkStatusDocumentV3:
     router_type = RouterStatusEntryMicroV3 if is_microdescriptor else RouterStatusEntryV3
   elif document_type == BridgeNetworkStatusDocument:
-    document_type, router_type = BridgeNetworkStatusDocument, RouterStatusEntryV2
+    document_type, router_type = BridgeNetworkStatusDocument, RouterStatusEntryBridgeV2
   elif document_type == DetachedSignature:
     yield document_type(document_file.read(), validate, **kwargs)
     return
@@ -1228,7 +1229,9 @@ class NetworkStatusDocumentV3(NetworkStatusDocument):
     self._footer(document_file, validate)
 
   def type_annotation(self):
-    if not self.is_microdescriptor:
+    if isinstance(self, BridgeNetworkStatusDocument):
+      return TypeAnnotation('bridge-network-status', 1, 0)
+    elif not self.is_microdescriptor:
       return TypeAnnotation('network-status-consensus-3' if not self.is_vote else 'network-status-vote-3', 1, 0)
     else:
       # Directory authorities do not issue a 'microdescriptor consensus' vote,
diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py
index 31913417..ce662b40 100644
--- a/stem/descriptor/router_status_entry.py
+++ b/stem/descriptor/router_status_entry.py
@@ -15,6 +15,8 @@ sources...
 
   RouterStatusEntry - Common parent for router status entries
     |- RouterStatusEntryV2 - Entry for a network status v2 document
+    |   +- RouterStatusEntryBridgeV2 - Entry for a bridge flavored v2 document
+    |
     |- RouterStatusEntryV3 - Entry for a network status v3 document
     +- RouterStatusEntryMicroV3 - Entry for a microdescriptor flavored v3 document
 """
@@ -540,6 +542,17 @@ class RouterStatusEntryV2(RouterStatusEntry):
     return ('r', 's', 'v')
 
 
+class RouterStatusEntryBridgeV2(RouterStatusEntryV2):
+  """
+  Information about an individual router stored within a bridge flavored
+  version 2 network status document.
+
+  .. versionadded:: 1.8.0
+  """
+
+  TYPE_ANNOTATION_NAME = 'bridge-network-status'
+
+
 class RouterStatusEntryV3(RouterStatusEntry):
   """
   Information about an individual router stored within a version 3 network
diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py
index 7d1f0205..3ab06f03 100644
--- a/test/unit/descriptor/collector.py
+++ b/test/unit/descriptor/collector.py
@@ -212,6 +212,21 @@ class TestCollector(unittest.TestCase):
 
   @patch('stem.util.connection.download')
   @patch('stem.descriptor.collector.CollecTor.files')
+  def test_reading_bridge_server_descriptors(self, files_mock, download_mock):
+    with open(get_resource('collector/bridge-server-descriptors-2019-02-cropped.tar'), 'rb') as archive:
+      download_mock.return_value = archive.read()
+
+    files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/server-descriptors/bridge-server-descriptors-2019-02.tar', 12345, '2016-09-04 09:21')]
+
+    descriptors = list(stem.descriptor.collector.get_server_descriptors(bridge = True))
+    self.assertEqual(4, len(descriptors))
+
+    f = descriptors[0]
+    self.assertEqual('BridgeDescriptor', type(f).__name__)
+    self.assertEqual('E90D1DE12B930DEC3F3E1127AAA25E47430CD3F4', f.fingerprint)
+
+  @patch('stem.util.connection.download')
+  @patch('stem.descriptor.collector.CollecTor.files')
   def test_reading_extrainfo_descriptors(self, files_mock, download_mock):
     with open(get_resource('collector/extra-infos-2019-04-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
@@ -227,6 +242,21 @@ class TestCollector(unittest.TestCase):
 
   @patch('stem.util.connection.download')
   @patch('stem.descriptor.collector.CollecTor.files')
+  def test_reading_bridge_extrainfo_descriptors(self, files_mock, download_mock):
+    with open(get_resource('collector/bridge-extra-infos-2019-03-cropped.tar'), 'rb') as archive:
+      download_mock.return_value = archive.read()
+
+    files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2019-03.tar', 12345, '2016-09-04 09:21')]
+
+    descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors(bridge = True))
+    self.assertEqual(6, len(descriptors))
+
+    f = descriptors[0]
+    self.assertEqual('BridgeExtraInfoDescriptor', type(f).__name__)
+    self.assertEqual('A0187027648A392C6AC413B66F7CD25DD001BF76', f.fingerprint)
+
+  @patch('stem.util.connection.download')
+  @patch('stem.descriptor.collector.CollecTor.files')
   def test_reading_microdescriptors(self, files_mock, download_mock):
     with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
@@ -267,14 +297,32 @@ class TestCollector(unittest.TestCase):
     self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(version = 2))))
     self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(microdescriptor = True))))
 
-    # but the microdescriptor archive *does* have microdescriptor consensuses
-
+  @patch('stem.util.connection.download')
+  @patch('stem.descriptor.collector.CollecTor.files')
+  def test_reading_microdescriptor_consensus(self, files_mock, download_mock):
     with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
 
+    files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/microdescs/microdescs-2019-05.tar', 12345, '2016-09-04 09:21')]
+
     descriptors = list(stem.descriptor.collector.get_consensus(microdescriptor = True))
     self.assertEqual(556, len(descriptors))
 
     f = descriptors[0]
     self.assertEqual('RouterStatusEntryMicroV3', type(f).__name__)
     self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint)
+
+  @patch('stem.util.connection.download')
+  @patch('stem.descriptor.collector.CollecTor.files')
+  def test_reading_bridge_consensus(self, files_mock, download_mock):
+    with open(get_resource('collector/bridge-statuses-2019-05-cropped.tar'), 'rb') as archive:
+      download_mock.return_value = archive.read()
+
+    files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/microdescs/bridge-statuses-2019-05.tar', 12345, '2016-09-04 09:21')]
+
+    descriptors = list(stem.descriptor.collector.get_consensus(bridge = True))
+    self.assertEqual(2593, len(descriptors))
+
+    f = descriptors[0]
+    self.assertEqual('RouterStatusEntryBridgeV2', type(f).__name__)
+    self.assertEqual('0035EA2A61E28D395F080ACA2244539490E70950', f.fingerprint)
diff --git a/test/unit/descriptor/data/collector/bridge-extra-infos-2019-03-cropped.tar b/test/unit/descriptor/data/collector/bridge-extra-infos-2019-03-cropped.tar
new file mode 100644
index 00000000..db9f3f7b
Binary files /dev/null and b/test/unit/descriptor/data/collector/bridge-extra-infos-2019-03-cropped.tar differ
diff --git a/test/unit/descriptor/data/collector/bridge-server-descriptors-2019-02-cropped.tar b/test/unit/descriptor/data/collector/bridge-server-descriptors-2019-02-cropped.tar
new file mode 100644
index 00000000..4d0255f3
Binary files /dev/null and b/test/unit/descriptor/data/collector/bridge-server-descriptors-2019-02-cropped.tar differ
diff --git a/test/unit/descriptor/data/collector/bridge-statuses-2019-05-cropped.tar b/test/unit/descriptor/data/collector/bridge-statuses-2019-05-cropped.tar
new file mode 100644
index 00000000..953d2a86
Binary files /dev/null and b/test/unit/descriptor/data/collector/bridge-statuses-2019-05-cropped.tar differ



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits