[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [stem/master] Fallback directory information
commit 332ef6e6fe89d9f0d48e9b95acff7de9f3ec647f
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date: Sun Feb 21 18:04:40 2016 -0800
Fallback directory information
Recently tor added fallback directory authorities to help clients bootstrap
when the dirauths are unavailable...
https://trac.torproject.org/projects/tor/ticket/17158
To help support this effort teor asked for a DocTor check to notify us when the
directories become unavailable...
https://trac.torproject.org/projects/tor/ticket/18177
Great idea but I'm expanding this. Stem now has a FallbackDirectory class with
two methods for getting this information...
* FallbackDirectory.from_remote() - Reads the latest fallback_dirs.inc from
gitweb, providing the latest fallback directories in tor's master branch.
* FallbackDirectory.from_cache() - Provides the latest fallback directories
Stem has cached. This is only as up-to-date as the Stem release you're
using but is quicker and avoids relying on gitweb.
Advantages are...
* Stem's descriptor.remote module now puts less load on the directory
authorities since it uses fallback directories as well.
* Running Stem's integ tests with the ONLINE target includes a test that
exercises all the fallback directories, notifying us if any are down.
---
cache_manual.py | 4 +-
docs/change_log.rst | 1 +
run_tests.py | 1 +
setup.py | 2 +-
stem/descriptor/remote.py | 266 ++++++++++++++++++++++++++++++++++++----
test/integ/descriptor/remote.py | 38 ++++++
test/unit/descriptor/remote.py | 56 +++++++++
7 files changed, 341 insertions(+), 27 deletions(-)
diff --git a/cache_manual.py b/cache_manual.py
index aa55dc5..19e73f0 100755
--- a/cache_manual.py
+++ b/cache_manual.py
@@ -6,7 +6,6 @@
Caches tor's latest manual content. Run this to pick new man page changes.
"""
-import os
import re
import sys
@@ -19,7 +18,6 @@ try:
except ImportError:
import urllib2 as urllib
-CACHE_PATH = os.path.join(os.path.dirname(__file__), 'stem', 'cached_tor_manual.cfg')
GITWEB_MAN_LOG = 'https://gitweb.torproject.org/tor.git/log/doc/tor.1.txt'
MAN_LOG_LINK = "href='/tor.git/commit/doc/tor.1.txt\?id=([^']*)'"
@@ -53,4 +51,4 @@ if __name__ == '__main__':
latest_manual.man_commit = man_commit
latest_manual.stem_commit = stem_commit
- latest_manual.save(CACHE_PATH)
+ latest_manual.save(stem.manual.CACHE_PATH)
diff --git a/docs/change_log.rst b/docs/change_log.rst
index 81bf5a2..c8dc434 100644
--- a/docs/change_log.rst
+++ b/docs/change_log.rst
@@ -58,6 +58,7 @@ The following are only available within Stem's `git repository
* **Descriptors**
+ * Fallback directory information, lessing the load of `stem.descriptor.remote <api/descriptor/remote.html>`_ on the directory authorities
* Support for ed25519 descriptor fields (:spec:`5a79d67`)
* Server descriptor validation fails with 'extra-info-digest line had an invalid value' from additions in proposal 228 (:trac:`16227`)
* :class:`~stem.descriptor.server_descriptor.BridgeDescriptor` now has 'ntor_onion_key' like its unsanitized counterparts
diff --git a/run_tests.py b/run_tests.py
index 3d0f845..2fb93bf 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -42,6 +42,7 @@ SRC_PATHS = [os.path.join(STEM_BASE, path) for path in (
'test',
'run_tests.py',
'cache_manual.py',
+ 'cache_fallback_directories.py',
'setup.py',
'tor-prompt',
os.path.join('docs', 'republish.py'),
diff --git a/setup.py b/setup.py
index 2a78b9b..2e492d4 100644
--- a/setup.py
+++ b/setup.py
@@ -17,5 +17,5 @@ distutils.core.setup(
packages = ['stem', 'stem.descriptor', 'stem.interpreter', 'stem.response', 'stem.util'],
keywords = 'tor onion controller',
scripts = ['tor-prompt'],
- package_data = {'stem': ['cached_tor_manual.cfg', 'settings.cfg'], 'stem.interpreter': ['settings.cfg'], 'stem.util': ['ports.cfg']},
+ package_data = {'stem': ['cached_tor_manual.cfg', 'settings.cfg'], 'stem.descriptor': ['fallback_directories.cfg'], 'stem.interpreter': ['settings.cfg'], 'stem.util': ['ports.cfg']},
)
diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py
index d199fb2..dc40e41 100644
--- a/stem/descriptor/remote.py
+++ b/stem/descriptor/remote.py
@@ -44,7 +44,11 @@ itself...
get_authorities - Provides tor directory information.
- DirectoryAuthority - Information about a tor directory authority.
+ Directory - Relay we can retrieve directory information from
+ |- DirectoryAuthority - Information about a tor directory authority
+ +- FallbackDirectory - Directory mirror tor uses when authories are unavailable
+ |- from_cache - Provides fallback directories cached with Stem.
+ +- from_remote - Retrieves fallback directories remotely from tor's latest commit.
Query - Asynchronous request to download tor descriptors
|- start - issues the query if it isn't already running
@@ -73,7 +77,9 @@ itself...
"""
import io
+import os
import random
+import re
import sys
import threading
import time
@@ -88,7 +94,7 @@ except ImportError:
import stem.descriptor
from stem import Flag
-from stem.util import log
+from stem.util import connection, log, tor_tools
# Tor has a limited number of descriptors we can fetch explicitly by their
# fingerprint or hashes due to a limit on the url length by squid proxies.
@@ -96,6 +102,9 @@ from stem.util import log
MAX_FINGERPRINTS = 96
MAX_MICRODESCRIPTOR_HASHES = 92
+GITWEB_FALLBACK_DIR_URL = 'https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc'
+CACHE_PATH = os.path.join(os.path.dirname(__file__), 'fallback_directories.cfg')
+
def _guess_descriptor_type(resource):
# Attempts to determine the descriptor type based on the resource url. This
@@ -340,8 +349,10 @@ class Query(object):
"""
if use_authority or not self.endpoints:
- authority = random.choice(list(filter(lambda auth: auth.v3ident is not None, get_authorities().values())))
- address, dirport = authority.address, authority.dir_port
+ directories = get_authorities().values() + FallbackDirectory.from_cache().values()
+
+ picked = random.choice(directories)
+ address, dirport = picked.address, picked.dir_port
else:
address, dirport = random.choice(self.endpoints)
@@ -390,8 +401,8 @@ class DescriptorDownloader(object):
def __init__(self, use_mirrors = False, **default_args):
self._default_args = default_args
- authorities = filter(lambda auth: auth.v3ident is not None, get_authorities().values())
- self._endpoints = [(auth.address, auth.dir_port) for auth in authorities]
+ directories = get_authorities().values() + FallbackDirectory.from_cache().values()
+ self._endpoints = [(directory.address, directory.dir_port) for directory in directories]
if use_mirrors:
try:
@@ -412,8 +423,8 @@ class DescriptorDownloader(object):
:raises: **Exception** if unable to determine the directory mirrors
"""
- authorities = filter(lambda auth: auth.v3ident is not None, get_authorities().values())
- new_endpoints = set([(auth.address, auth.dir_port) for auth in authorities])
+ directories = get_authorities().values() + FallbackDirectory.from_cache().values()
+ new_endpoints = set([(directory.address, directory.dir_port) for directory in directories])
consensus = list(self.get_consensus(document_handler = stem.descriptor.DocumentHandler.DOCUMENT).run())[0]
@@ -617,7 +628,39 @@ class DescriptorDownloader(object):
)
-class DirectoryAuthority(object):
+class Directory(object):
+ """
+ Relay we can constact for directory information
+
+ .. versionadded:: 1.5.0
+
+ :var str address: IP address of the authority, currently they're all IPv4 but
+ this may not always be the case
+ :var int or_port: port on which the relay services relay traffic
+ :var int dir_port: port on which directory information is available
+ :var str fingerprint: relay fingerprint
+ :var str nickname: nickname of the authority
+ """
+
+ def __init__(self, address, or_port, dir_port, fingerprint, nickname):
+ self.address = address
+ self.or_port = or_port
+ self.dir_port = dir_port
+ self.fingerprint = fingerprint
+ self.nickname = nickname
+
+ def __eq__(self, other):
+ if not isinstance(other, Directory):
+ return False
+
+ for attr in ('nickname', 'address', 'or_port', 'dir_port', 'fingerprint'):
+ if getattr(self, attr) != getattr(other, attr):
+ return False
+
+ return True
+
+
+class DirectoryAuthority(Directory):
"""
Tor directory authority, a special type of relay `hardcoded into tor
<https://gitweb.torproject.org/tor.git/tree/src/or/config.c#n819>`_
@@ -648,23 +691,21 @@ class DirectoryAuthority(object):
.. versionchanged:: 1.3.0
Added the is_bandwidth_authority attribute.
- :var str nickname: nickname of the authority
- :var str address: IP address of the authority, currently they're all IPv4 but
- this may not always be the case
- :var int or_port: port on which the relay services relay traffic
- :var int dir_port: port on which directory information is available
- :var str fingerprint: relay fingerprint
:var str v3ident: identity key fingerprint used to sign votes and consensus
+ :var bool is_bandwidth_authority: **True** if this is a bandwidth authority,
+ **False** otherwise
"""
- def __init__(self, nickname = None, address = None, or_port = None, dir_port = None, is_bandwidth_authority = False, fingerprint = None, v3ident = None):
- self.nickname = nickname
- self.address = address
- self.or_port = or_port
- self.dir_port = dir_port
- self.is_bandwidth_authority = is_bandwidth_authority
- self.fingerprint = fingerprint
+ def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None, v3ident = None, is_bandwidth_authority = False):
+ super(DirectoryAuthority, self).__init__(address, or_port, dir_port, fingerprint, nickname)
self.v3ident = v3ident
+ self.is_bandwidth_authority = is_bandwidth_authority
+
+ def __eq__(self, other):
+ if isinstance(other, DirectoryAuthority) and super(DirectoryAuthority, self).__eq__(other):
+ return self.v3ident == other.v3ident and self.is_bandwidth_authority == other.is_bandwidth_authority
+ else:
+ return False
DIRECTORY_AUTHORITIES = {
@@ -767,7 +808,186 @@ def get_authorities():
The directory information hardcoded into Tor and occasionally changes, so the
information this provides might not necessarily match your version of tor.
- :returns: dict of str nicknames to :class:`~stem.descriptor.remote.DirectoryAuthority` instances
+ :returns: **dict** of **str** nicknames to :class:`~stem.descriptor.remote.DirectoryAuthority` instances
"""
return dict(DIRECTORY_AUTHORITIES)
+
+
+class FallbackDirectory(Directory):
+ """
+ Tor directories tor uses as alternates for the authorities. These relays are
+ `hardcoded in tor <https://gitweb.torproject.org/tor.git/tree/src/or/fallback_dirs.inc>`_.
+
+ .. versionadded:: 1.5.0
+ """
+
+ def __init__(self, address = None, or_port = None, dir_port = None, fingerprint = None, nickname = None):
+ super(FallbackDirectory, self).__init__(address, or_port, dir_port, fingerprint, nickname)
+
+ @staticmethod
+ def from_cache():
+ """
+ Provides fallback directory information cached with Stem. Unlike
+ :func:`~stem.descriptor.remote.FallbackDirectory.from_remote` this doesn't
+ have any system requirements, and is faster too. Only drawback is that
+ these fallback directories are only as up to date as the Stem release we're
+ using.
+
+ :returns: **dict** of **str** fingerprints to their
+ :class:`~stem.descriptor.remote.FallbackDirectory`
+ """
+
+ conf = stem.util.conf.Config()
+ conf.load(CACHE_PATH)
+
+ results = {}
+
+ for nickname in set([key.split('.')[0] for key in conf.keys()]):
+ if nickname in ('tor_commit', 'stem_commit'):
+ continue
+
+ attr = {}
+
+ for attr_name in ('address', 'or_port', 'dir_port', 'fingerprint'):
+ key = '%s.%s' % (nickname, attr_name)
+ attr[attr_name] = conf.get(key)
+
+ if not attr[attr_name]:
+ raise IOError("'%s' is missing from %s" % (key, CACHE_PATH))
+
+ if not connection.is_valid_ipv4_address(attr['address']):
+ raise IOError("'%s.address' was an invalid address (%s)" % (nickname, attr['address']))
+ elif not connection.is_valid_port(attr['or_port']):
+ raise IOError("'%s.or_port' was an invalid port (%s)" % (nickname, attr['or_port']))
+ elif not connection.is_valid_port(attr['dir_port']):
+ raise IOError("'%s.dir_port' was an invalid port (%s)" % (nickname, attr['dir_port']))
+ elif not tor_tools.is_valid_fingerprint(attr['fingerprint']):
+ raise IOError("'%s.fingerprint' was an invalid fingerprint (%s)" % (nickname, attr['fingerprint']))
+
+ results[attr['fingerprint']] = FallbackDirectory(
+ address = attr['address'],
+ or_port = int(attr['or_port']),
+ dir_port = int(attr['dir_port']),
+ fingerprint = attr['fingerprint'],
+ nickname = nickname,
+ )
+
+ return results
+
+ @staticmethod
+ def from_remote(timeout = 60):
+ """
+ Reads and parses tor's latest fallback directories `from
+ gitweb.torproject.org
+ <https://gitweb.torproject.org/tor.git/plain/src/or/fallback_dirs.inc>`_.
+ Note that while convenient, this reliance on GitWeb means you should alway
+ call with a fallback, such as...
+
+ ::
+
+ try:
+ fallback_directories = stem.descriptor.remote.from_remote()
+ except IOError:
+ fallback_directories = stem.descriptor.remote.from_cache()
+
+ :param int timeout: seconds to wait before timing out the request
+
+ :returns: **dict** of **str** fingerprints to their
+ :class:`~stem.descriptor.remote.FallbackDirectory`
+
+ :raises: **IOError** if unable to retrieve the fallback directories
+ """
+
+ try:
+ fallback_dir_page = urllib.urlopen(GITWEB_FALLBACK_DIR_URL, timeout = timeout).read()
+ except:
+ exc = sys.exc_info()[1]
+ raise IOError("Unable to download tor's fallback directories from %s: %s" % (GITWEB_FALLBACK_DIR_URL, exc))
+
+ # Example of an entry...
+ #
+ # /*
+ # wagner
+ # Flags: Fast Guard Running Stable V2Dir Valid
+ # Fallback Weight: 43680 / 491920 (8.879%)
+ # Consensus Weight: 62600 / 546000 (11.465%)
+ # Rarely used email <trff914 AT gmail DOT com>
+ # */
+ # "5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33"
+ # " weight=43680",
+
+ results, nickname, last_line = {}, None, None
+
+ for line in fallback_dir_page.splitlines():
+ if last_line == '/*':
+ nickname = line
+ elif line.startswith('"'):
+ addr_line_match = re.match('"([\d\.]+):(\d+) orport=(\d+) id=([\dA-F]{40}).*', line)
+
+ if addr_line_match:
+ address, dir_port, or_port, fingerprint = addr_line_match.groups()
+
+ if not connection.is_valid_ipv4_address(address):
+ raise IOError('%s has an invalid address: %s' % (nickname, address))
+ elif not connection.is_valid_port(or_port):
+ raise IOError('%s has an invalid or_port: %s' % (nickname, or_port))
+ elif not connection.is_valid_port(dir_port):
+ raise IOError('%s has an invalid dir_port: %s' % (nickname, dir_port))
+ elif not tor_tools.is_valid_fingerprint(fingerprint):
+ raise IOError('%s has an invalid fingerprint: %s' % (nickname, fingerprint))
+
+ results[fingerprint] = FallbackDirectory(
+ address = address,
+ or_port = int(or_port),
+ dir_port = int(dir_port),
+ fingerprint = fingerprint,
+ nickname = nickname,
+ )
+
+ last_line = line
+
+ return results
+
+
+def _fallback_directory_differences(previous_directories, new_directories):
+ """
+ Provides a description of how fallback directories differ.
+ """
+
+ lines = []
+
+ added_fp = set(new_directories.keys()).difference(previous_directories.keys())
+ removed_fp = set(previous_directories.keys()).difference(new_directories.keys())
+
+ for fp in added_fp:
+ directory = new_directories[fp]
+
+ lines += [
+ '* Added %s as a new fallback directory:' % directory.nickname,
+ ' address: %s' % directory.address,
+ ' or_port: %s' % directory.or_port,
+ ' dir_port: %s' % directory.dir_port,
+ ' fingerprint: %s' % directory.fingerprint,
+ '',
+ ]
+
+ for fp in removed_fp:
+ lines.append('* Removed %s as a fallback directory' % previous_directories[fp].nickname)
+
+ for fp in new_directories:
+ if fp in added_fp or fp in removed_fp:
+ continue # already discussed these
+
+ previous_directory = previous_directories[fp]
+ new_directory = new_directories[fp]
+
+ if previous_directory != new_directory:
+ for attr in ('nickname', 'address', 'or_port', 'dir_port', 'fingerprint'):
+ old_attr = getattr(previous_directory, attr)
+ new_attr = getattr(new_directory, attr)
+
+ if old_attr != new_attr:
+ lines.append('* Changed the %s of %s from %s to %s' % (attr, fp, old_attr, new_attr))
+
+ return '\n'.join(lines)
diff --git a/test/integ/descriptor/remote.py b/test/integ/descriptor/remote.py
index fd37b57..9ce9b8f 100644
--- a/test/integ/descriptor/remote.py
+++ b/test/integ/descriptor/remote.py
@@ -218,3 +218,41 @@ class TestDescriptorDownloader(unittest.TestCase):
self.assertTrue(isinstance(single_query_results[0], stem.descriptor.networkstatus.KeyCertificate))
self.assertEqual(2, len(list(multiple_query)))
+
+ @require_online
+ def test_that_cache_is_up_to_date(self):
+ """
+ Check if the cached fallback directories bundled with Stem are up to date
+ or not.
+ """
+
+ cached_fallback_directories = stem.descriptor.remote.FallbackDirectory.from_cache()
+ latest_fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote()
+
+ if cached_fallback_directories != latest_fallback_directories:
+ self.fail("Stem's cached fallback directories are out of date. Please run 'cache_fallback_directories.py'...\n\n%s" % stem.descriptor.remote._fallback_directory_differences(cached_fallback_directories, latest_fallback_directories))
+
+ @require_online
+ def test_that_fallback_directories_are_reachable(self):
+ """
+ Fetch information from each fallback directory to confirm that it's
+ available.
+ """
+
+ unsuccessful = {}
+ downloader = stem.descriptor.remote.DescriptorDownloader()
+ moria1_v3ident = stem.descriptor.remote.get_authorities()['moria1'].v3ident
+
+ for fallback_directory in stem.descriptor.remote.FallbackDirectory.from_cache().values():
+ try:
+ downloader.get_key_certificates(authority_v3idents = moria1_v3ident, endpoints = [(fallback_directory.address, fallback_directory.dir_port)]).run()
+ except Exception as exc:
+ unsuccessful[fallback_directory] = exc
+
+ if unsuccessful:
+ lines = ['We were unable to contact the following fallback directories...\n']
+
+ for fallback_directory, exc in unsuccessful.items():
+ lines.append('* %s:%s (%s): %s' % (fallback_directory.address, fallback_directory.dir_port, fallback_directory.fingerprint, exc))
+
+ self.fail('\n'.join(lines))
diff --git a/test/unit/descriptor/remote.py b/test/unit/descriptor/remote.py
index 19907a7..f0e0dea 100644
--- a/test/unit/descriptor/remote.py
+++ b/test/unit/descriptor/remote.py
@@ -58,6 +58,32 @@ iO3EUE0AEYah2W9gdz8t+i3Dtr0zgqLS841GC/TyDKCm+MKmN8d098qnwK0NGF9q
-----END SIGNATURE-----
"""
+FALLBACK_DIR_CONTENT = b"""\
+/* Trial fallbacks for 0.2.8.1-alpha with ADDRESS_AND_PORT_STABLE_DAYS = 30
+ * This works around an issue where relays post a descriptor without a DirPort
+ * when restarted. If these relays stay up, they will have been up for 120 days
+ * by the 0.2.8 stable release -- teor */
+/*
+wagner
+Flags: Fast Guard Running Stable V2Dir Valid
+Fallback Weight: 43680 / 491920 (8.879%)
+Consensus Weight: 62600 / 546000 (11.465%)
+Rarely used email <trff914 AT gmail DOT com>
+*/
+"5.175.233.86:80 orport=443 id=5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33"
+" weight=43680",
+/*
+kitten2
+Flags: Fast Guard HSDir Running Stable V2Dir Valid
+Fallback Weight: 43680 / 491920 (8.879%)
+Consensus Weight: 59100 / 546000 (10.824%)
+0xEFB74277ECE4E222 Aeris <aeris+tor AT imirhil DOT fr> - 1aerisnnLWPchhDSXpxWGYWwLiSFUVFnd
+*/
+"62.210.124.124:9130 orport=9101 id=2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E"
+" ipv6=[2001:bc8:3f23:100::1]:9101"
+" weight=43680",
+"""
+
class TestDescriptorDownloader(unittest.TestCase):
@patch(URL_OPEN)
@@ -154,3 +180,33 @@ class TestDescriptorDownloader(unittest.TestCase):
self.assertEqual(1, len(list(query)))
self.assertEqual(1, len(list(query)))
self.assertEqual(1, len(list(query)))
+
+ def test_fallback_directories_from_cache(self):
+ # quick sanity test that we can load cached content
+ fallback_directories = stem.descriptor.remote.FallbackDirectory.from_cache()
+ self.assertTrue(len(fallback_directories) > 10)
+ self.assertEqual('wagner', fallback_directories['5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33'].nickname)
+
+ @patch(URL_OPEN)
+ def test_fallback_directories_from_remote(self, urlopen_mock):
+ urlopen_mock.return_value = io.BytesIO(FALLBACK_DIR_CONTENT)
+ fallback_directories = stem.descriptor.remote.FallbackDirectory.from_remote()
+
+ expected = {
+ '5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33': stem.descriptor.remote.FallbackDirectory(
+ nickname = 'wagner',
+ address = '5.175.233.86',
+ or_port = 443,
+ dir_port = 80,
+ fingerprint = '5525D0429BFE5DC4F1B0E9DE47A4CFA169661E33',
+ ),
+ '2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E': stem.descriptor.remote.FallbackDirectory(
+ nickname = 'kitten2',
+ address = '62.210.124.124',
+ or_port = 9101,
+ dir_port = 9130,
+ fingerprint = '2EBD117806EE43C3CC885A8F1E4DC60F207E7D3E',
+ ),
+ }
+
+ self.assertEqual(expected, fallback_directories)
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits