[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [stem/master] NetworkStatusDocumentV2 lazy loading
commit 0f1b71125ab2709a041c02e0b700d47c4d69b6d7
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date: Sun Jan 18 11:04:00 2015 -0800
NetworkStatusDocumentV2 lazy loading
---
stem/descriptor/__init__.py | 14 ++-
stem/descriptor/microdescriptor.py | 3 +-
stem/descriptor/networkstatus.py | 215 +++++++++++++---------------------
stem/descriptor/server_descriptor.py | 3 +-
4 files changed, 98 insertions(+), 137 deletions(-)
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index cd9dcde..9a3412d 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -319,6 +319,13 @@ def _values(line, entries):
return [entry[0] for entry in entries[line]]
+def _parse_simple_line(keyword, attribute):
+ def _parse(descriptor, entries):
+ setattr(descriptor, attribute, _value(keyword, entries))
+
+ return _parse
+
+
def _parse_timestamp_line(keyword, attribute):
# "<keyword>" YYYY-MM-DD HH:MM:SS
@@ -347,15 +354,18 @@ def _parse_forty_character_hex(keyword, attribute):
return _parse
-def _parse_key_block(keyword, attribute, expected_block_type):
+def _parse_key_block(keyword, attribute, expected_block_type, value_attribute = None):
def _parse(descriptor, entries):
value, block_type, block_contents = entries[keyword][0]
if not block_contents or block_type != expected_block_type:
- raise ValueError("'%s' should be followed by a %s block" % (keyword, expected_block_type))
+ raise ValueError("'%s' should be followed by a %s block, but was a %s" % (keyword, expected_block_type, block_type))
setattr(descriptor, attribute, block_contents)
+ if value_attribute:
+ setattr(descriptor, value_attribute, value)
+
return _parse
diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py
index 0590cbb..fd9ef9b 100644
--- a/stem/descriptor/microdescriptor.py
+++ b/stem/descriptor/microdescriptor.py
@@ -75,6 +75,7 @@ from stem.descriptor import (
_read_until_keywords,
_value,
_values,
+ _parse_simple_line,
_parse_key_block,
)
@@ -171,7 +172,7 @@ def _parse_id_line(descriptor, entries):
_parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY')
-_parse_ntor_onion_key_line = lambda descriptor, entries: setattr(descriptor, 'ntor_onion_key', _value('ntor-onion-key', entries))
+_parse_ntor_onion_key_line = _parse_simple_line('ntor-onion-key', 'ntor_onion_key')
_parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', _value('family', entries).split(' '))
_parse_p_line = lambda descriptor, entries: stem.descriptor.router_status_entry._parse_p_line(descriptor, _value('p', entries), True)
_parse_p6_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('p6', entries)))
diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py
index 70f325b..29d2593 100644
--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -63,6 +63,7 @@ from stem.descriptor import (
_get_descriptor_components,
_read_until_keywords,
_value,
+ _parse_simple_line,
_parse_timestamp_line,
_parse_forty_character_hex,
_parse_key_block,
@@ -269,12 +270,51 @@ class NetworkStatusDocument(Descriptor):
Common parent for network status documents.
"""
- def __init__(self, raw_content):
- super(NetworkStatusDocument, self).__init__(raw_content)
- self._unrecognized_lines = []
- def get_unrecognized_lines(self):
- return list(self._unrecognized_lines)
+def _parse_version_line(keyword, attribute, expected_version):
+ def _parse(descriptor, entries):
+ value = _value(keyword, entries)
+
+ if not value.isdigit():
+ raise ValueError('Document has a non-numeric version: %s %s' % (keyword, value))
+
+ setattr(descriptor, attribute, int(value))
+
+ if int(value) != expected_version:
+ raise ValueError("Expected a version %i document, but got version '%s' instead" % (expected_version, value))
+
+ return _parse
+
+
+def _parse_dir_source_line(descriptor, entries):
+ value = _value('dir-source', entries)
+ dir_source_comp = value.split()
+
+ if len(dir_source_comp) < 3:
+ raise ValueError("The 'dir-source' line of a v2 network status document must have three values: dir-source %s" % value)
+
+ if not dir_source_comp[0]:
+ # https://trac.torproject.org/7055
+ raise ValueError("Authority's hostname can't be blank: dir-source %s" % value)
+ elif not stem.util.connection.is_valid_ipv4_address(dir_source_comp[1]):
+ raise ValueError("Authority's address isn't a valid IPv4 address: %s" % dir_source_comp[1])
+ elif not stem.util.connection.is_valid_port(dir_source_comp[2], allow_zero = True):
+ raise ValueError("Authority's DirPort is invalid: %s" % dir_source_comp[2])
+
+ descriptor.hostname = dir_source_comp[0]
+ descriptor.address = dir_source_comp[1]
+ descriptor.dir_port = None if dir_source_comp[2] == '0' else int(dir_source_comp[2])
+
+
+_parse_network_status_version_line = _parse_version_line('network-status-version', 'version', 2)
+_parse_fingerprint_line = _parse_forty_character_hex('fingerprint', 'fingerprint')
+_parse_contact_line = _parse_simple_line('contact', 'contact')
+_parse_dir_signing_key_line = _parse_key_block('dir-signing-key', 'signing_key', 'RSA PUBLIC KEY')
+_parse_client_versions_line = lambda descriptor, entries: setattr(descriptor, 'client_versions', _value('client-versions', entries).split(','))
+_parse_server_versions_line = lambda descriptor, entries: setattr(descriptor, 'server_versions', _value('server-versions', entries).split(','))
+_parse_published_line = _parse_timestamp_line('published', 'published')
+_parse_dir_options_line = lambda descriptor, entries: setattr(descriptor, 'options', _value('dir-options', entries).split())
+_parse_directory_signature_line = _parse_key_block('directory-signature', 'signature', 'SIGNATURE', value_attribute = 'signing_authority')
class NetworkStatusDocumentV2(NetworkStatusDocument):
@@ -306,24 +346,39 @@ class NetworkStatusDocumentV2(NetworkStatusDocument):
a default value, others are left as **None** if undefined
"""
- def __init__(self, raw_content, validate = True):
- super(NetworkStatusDocumentV2, self).__init__(raw_content)
+ ATTRIBUTES = {
+ 'version': (None, _parse_network_status_version_line),
+ 'hostname': (None, _parse_dir_source_line),
+ 'address': (None, _parse_dir_source_line),
+ 'dir_port': (None, _parse_dir_source_line),
+ 'fingerprint': (None, _parse_fingerprint_line),
+ 'contact': (None, _parse_contact_line),
+ 'signing_key': (None, _parse_dir_signing_key_line),
- self.version = None
- self.hostname = None
- self.address = None
- self.dir_port = None
- self.fingerprint = None
- self.contact = None
- self.signing_key = None
+ 'client_versions': ([], _parse_client_versions_line),
+ 'server_versions': ([], _parse_server_versions_line),
+ 'published': (None, _parse_published_line),
+ 'options': ([], _parse_dir_options_line),
- self.client_versions = []
- self.server_versions = []
- self.published = None
- self.options = []
+ 'signing_authority': (None, _parse_directory_signature_line),
+ 'signatures': (None, _parse_directory_signature_line),
+ }
+
+ PARSER_FOR_LINE = {
+ 'network-status-version': _parse_network_status_version_line,
+ 'dir-source': _parse_dir_source_line,
+ 'fingerprint': _parse_fingerprint_line,
+ 'contact': _parse_contact_line,
+ 'dir-signing-key': _parse_dir_signing_key_line,
+ 'client-versions': _parse_client_versions_line,
+ 'server-versions': _parse_server_versions_line,
+ 'published': _parse_published_line,
+ 'dir-options': _parse_dir_options_line,
+ 'directory-signature': _parse_directory_signature_line,
+ }
- self.signing_authority = None
- self.signatures = None
+ def __init__(self, raw_content, validate = True):
+ super(NetworkStatusDocumentV2, self).__init__(raw_content, lazy_load = not validate)
# Splitting the document from the routers. Unlike v3 documents we're not
# bending over backwards on the validation by checking the field order or
@@ -351,96 +406,15 @@ class NetworkStatusDocumentV2(NetworkStatusDocument):
if validate:
self._check_constraints(entries)
+ self._parse(entries, validate)
- self._parse(entries, validate)
-
- def _parse(self, entries, validate):
- for keyword, values in list(entries.items()):
- value, block_type, block_contents = values[0]
-
- line = '%s %s' % (keyword, value) # original line
-
- if block_contents:
- line += '\n%s' % block_contents
-
- if keyword == 'network-status-version':
- if not value.isdigit():
- if not validate:
- continue
-
- raise ValueError('Network status document has a non-numeric version: %s' % line)
-
- self.version = int(value)
-
- if validate and self.version != 2:
- raise ValueError("Expected a version 2 network status document, got version '%s' instead" % self.version)
- elif keyword == 'dir-source':
- dir_source_comp = value.split()
-
- if len(dir_source_comp) < 3:
- if not validate:
- continue
-
- raise ValueError("The 'dir-source' line of a v2 network status document must have three values: %s" % line)
-
- if validate:
- if not dir_source_comp[0]:
- # https://trac.torproject.org/7055
- raise ValueError("Authority's hostname can't be blank: %s" % line)
- elif not stem.util.connection.is_valid_ipv4_address(dir_source_comp[1]):
- raise ValueError("Authority's address isn't a valid IPv4 address: %s" % dir_source_comp[1])
- elif not stem.util.connection.is_valid_port(dir_source_comp[2], allow_zero = True):
- raise ValueError("Authority's DirPort is invalid: %s" % dir_source_comp[2])
- elif not dir_source_comp[2].isdigit():
- continue
-
- self.hostname = dir_source_comp[0]
- self.address = dir_source_comp[1]
- self.dir_port = None if dir_source_comp[2] == '0' else int(dir_source_comp[2])
- elif keyword == 'fingerprint':
- if validate and not stem.util.tor_tools.is_valid_fingerprint(value):
- raise ValueError("Authority's fingerprint in a v2 network status document is malformed: %s" % line)
-
- self.fingerprint = value
- elif keyword == 'contact':
- self.contact = value
- elif keyword == 'dir-signing-key':
- if validate and (not block_contents or block_type != 'RSA PUBLIC KEY'):
- raise ValueError("'dir-signing-key' should be followed by a RSA PUBLIC KEY block: %s" % line)
-
- self.signing_key = block_contents
- elif keyword in ('client-versions', 'server-versions'):
- # v2 documents existed while there were tor versions using the 'old'
- # style, hence we aren't attempting to parse them
-
- for version_str in value.split(','):
- if keyword == 'client-versions':
- self.client_versions.append(version_str)
- elif keyword == 'server-versions':
- self.server_versions.append(version_str)
- elif keyword == 'published':
- try:
- self.published = stem.util.str_tools._parse_timestamp(value)
- except ValueError:
- if validate:
- raise ValueError("Version 2 network status document's 'published' time wasn't parsable: %s" % value)
- elif keyword == 'dir-options':
- self.options = value.split()
- elif keyword == 'directory-signature':
- if validate and (not block_contents or block_type != 'SIGNATURE'):
- raise ValueError("'directory-signature' should be followed by a SIGNATURE block: %s" % line)
-
- self.signing_authority = value
- self.signature = block_contents
- else:
- self._unrecognized_lines.append(line)
-
- # 'client-versions' and 'server-versions' are only required if 'Versions'
- # is among the options
+ # 'client-versions' and 'server-versions' are only required if 'Versions'
+ # is among the options
- if validate and 'Versions' in self.options:
- if not ('client-versions' in entries and 'server-versions' in entries):
+ if 'Versions' in self.options and not ('client-versions' in entries and 'server-versions' in entries):
raise ValueError("Version 2 network status documents must have a 'client-versions' and 'server-versions' when 'Versions' is listed among its dir-options:\n%s" % str(self))
+ else:
+ self._entries = entries
def _check_constraints(self, entries):
required_fields = [field for (field, is_mandatory) in NETWORK_STATUS_V2_FIELDS if is_mandatory]
@@ -1060,7 +1034,6 @@ def _parse_dir_source_line(descriptor, entries):
descriptor.is_legacy = descriptor.nickname.endswith('-legacy')
-_parse_contact_line = lambda descriptor, entries: setattr(descriptor, 'contact', _value('contact', entries))
_parse_legacy_dir_key_line = _parse_forty_character_hex('legacy-dir-key', 'legacy_dir_key')
_parse_vote_digest_line = _parse_forty_character_hex('vote-digest', 'vote_digest')
@@ -1207,20 +1180,6 @@ class DirectoryAuthority(Descriptor):
return self._compare(other, lambda s, o: s <= o)
-def _parse_dir_key_certificate_version_line(descriptor, entries):
- # "dir-key-certificate-version" version
-
- value = _value('dir-key-certificate-version', entries)
-
- if not value.isdigit():
- raise ValueError('Key certificate has a non-integer version: dir-key-certificate-version %s' % value)
-
- descriptor.version = int(value)
-
- if descriptor.version != 3:
- raise ValueError("Expected a version 3 key certificate, got version '%i' instead" % descriptor.version)
-
-
def _parse_dir_address_line(descriptor, entries):
# "dir-address" IPPort
@@ -1240,17 +1199,7 @@ def _parse_dir_address_line(descriptor, entries):
descriptor.dir_port = int(dirport)
-def _parse_fingerprint_line(descriptor, entries):
- # "fingerprint" fingerprint
-
- value = _value('fingerprint', entries)
-
- if not stem.util.tor_tools.is_valid_fingerprint(value):
- raise ValueError("Key certificate's fingerprint is malformed: fingerprint %s" % value)
-
- descriptor.fingerprint = value
-
-
+_parse_dir_key_certificate_version_line = _parse_version_line('dir-key-certificate-version', 'version', 3)
_parse_dir_key_published_line = _parse_timestamp_line('dir-key-published', 'published')
_parse_dir_key_expires_line = _parse_timestamp_line('dir-key-expires', 'expires')
_parse_identity_key_line = _parse_key_block('dir-identity-key', 'identity_key', 'RSA PUBLIC KEY')
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py
index f626050..1cdf9be 100644
--- a/stem/descriptor/server_descriptor.py
+++ b/stem/descriptor/server_descriptor.py
@@ -56,6 +56,7 @@ from stem.descriptor import (
_read_until_keywords,
_value,
_values,
+ _parse_simple_line,
_parse_timestamp_line,
_parse_forty_character_hex,
_parse_key_block,
@@ -378,7 +379,7 @@ _parse_eventdns_line = lambda descriptor, entries: setattr(descriptor, 'eventdns
_parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY')
_parse_signing_key_line = _parse_key_block('signing-key', 'signing_key', 'RSA PUBLIC KEY')
_parse_router_signature_line = _parse_key_block('router-signature', 'signature', 'SIGNATURE')
-_parse_ntor_onion_key_line = lambda descriptor, entries: setattr(descriptor, 'ntor_onion_key', _value('ntor-onion-key', entries))
+_parse_ntor_onion_key_line = _parse_simple_line('ntor-onion-key', 'ntor_onion_key')
_parse_router_digest_line = _parse_forty_character_hex('router-digest', '_digest')
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits