[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [stem/master] Microdescriptor lazy loading
commit f19d87ab0f16928be370114c993f512b3d2dfac5
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date: Sat Jan 17 17:36:54 2015 -0800
Microdescriptor lazy loading
Simplest descriptor type so pretty simple switch.
---
stem/descriptor/__init__.py | 4 +-
stem/descriptor/extrainfo_descriptor.py | 3 +-
stem/descriptor/microdescriptor.py | 115 ++++++++++++++-----------------
stem/descriptor/server_descriptor.py | 6 +-
4 files changed, 56 insertions(+), 72 deletions(-)
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index bc1b462..ef96dd7 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -351,11 +351,11 @@ class Descriptor(object):
ATTRIBUTES = {} # mapping of 'attribute' => (default_value, parsing_function)
PARSER_FOR_LINE = {} # line keyword to its associated parsing function
- def __init__(self, contents):
+ def __init__(self, contents, lazy_load = False):
self._path = None
self._archive_path = None
self._raw_contents = contents
- self._lazy_loading = False
+ self._lazy_loading = lazy_load
self._unrecognized_lines = []
def get_path(self):
diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py
index 9e86798..4137e47 100644
--- a/stem/descriptor/extrainfo_descriptor.py
+++ b/stem/descriptor/extrainfo_descriptor.py
@@ -786,11 +786,10 @@ class ExtraInfoDescriptor(Descriptor):
:raises: **ValueError** if the contents is malformed and validate is True
"""
- super(ExtraInfoDescriptor, self).__init__(raw_contents)
+ super(ExtraInfoDescriptor, self).__init__(raw_contents, lazy_load = not validate)
raw_contents = stem.util.str_tools._to_unicode(raw_contents)
entries = _get_descriptor_components(raw_contents, validate)
- self._lazy_loading = not validate
if validate:
for keyword in self._required_fields():
diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py
index 093e6f0..0590cbb 100644
--- a/stem/descriptor/microdescriptor.py
+++ b/stem/descriptor/microdescriptor.py
@@ -73,6 +73,9 @@ from stem.descriptor import (
Descriptor,
_get_descriptor_components,
_read_until_keywords,
+ _value,
+ _values,
+ _parse_key_block,
)
try:
@@ -151,6 +154,29 @@ def _parse_file(descriptor_file, validate = True, **kwargs):
break # done parsing descriptors
+def _parse_a_line(descriptor, entries):
+ for value in _values('a', entries):
+ stem.descriptor.router_status_entry._parse_a_line(descriptor, value, True)
+
+
+def _parse_id_line(descriptor, entries):
+ value = _value('id', entries)
+ value_comp = value.split()
+
+ if len(value_comp) >= 2:
+ descriptor.identifier_type = value_comp[0]
+ descriptor.identifier = value_comp[1]
+ else:
+ raise ValueError("'id' lines should contain both the key type and digest: id %s" % value)
+
+
+_parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY')
+_parse_ntor_onion_key_line = lambda descriptor, entries: setattr(descriptor, 'ntor_onion_key', _value('ntor-onion-key', entries))
+_parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', _value('family', entries).split(' '))
+_parse_p_line = lambda descriptor, entries: stem.descriptor.router_status_entry._parse_p_line(descriptor, _value('p', entries), True)
+_parse_p6_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('p6', entries)))
+
+
class Microdescriptor(Descriptor):
"""
Microdescriptor (`descriptor specification
@@ -173,33 +199,41 @@ class Microdescriptor(Descriptor):
**\*** attribute is required when we're parsed with validation
"""
+ ATTRIBUTES = {
+ 'onion_key': (None, _parse_onion_key_line),
+ 'ntor_onion_key': (None, _parse_ntor_onion_key_line),
+ 'or_addresses': ([], _parse_a_line),
+ 'family': ([], _parse_family_line),
+ 'exit_policy': (stem.exit_policy.MicroExitPolicy('reject 1-65535'), _parse_p_line),
+ 'exit_policy_v6': (None, _parse_p6_line),
+ 'identifier_type': (None, _parse_id_line),
+ 'identifier': (None, _parse_id_line),
+ }
+
+ PARSER_FOR_LINE = {
+ 'onion-key': _parse_onion_key_line,
+ 'ntor-onion-key': _parse_ntor_onion_key_line,
+ 'a': _parse_a_line,
+ 'family': _parse_family_line,
+ 'p': _parse_p_line,
+ 'p6': _parse_p6_line,
+ 'id': _parse_id_line,
+ }
+
def __init__(self, raw_contents, validate = True, annotations = None):
- super(Microdescriptor, self).__init__(raw_contents)
+ super(Microdescriptor, self).__init__(raw_contents, lazy_load = not validate)
raw_contents = stem.util.str_tools._to_unicode(raw_contents)
self.digest = hashlib.sha256(self.get_bytes()).hexdigest().upper()
-
- self.onion_key = None
- self.ntor_onion_key = None
- self.or_addresses = []
- self.family = []
- self.exit_policy = stem.exit_policy.MicroExitPolicy('reject 1-65535')
- self.exit_policy_v6 = None
- self.identifier_type = None
- self.identifier = None
-
- self._unrecognized_lines = []
-
self._annotation_lines = annotations if annotations else []
entries = _get_descriptor_components(raw_contents, validate)
- self._parse(entries, validate)
if validate:
+ self._parse(entries, validate)
self._check_constraints(entries)
-
- def get_unrecognized_lines(self):
- return list(self._unrecognized_lines)
+ else:
+ self._entries = entries
@lru_cache()
def get_annotations(self):
@@ -237,53 +271,6 @@ class Microdescriptor(Descriptor):
return self._annotation_lines
- def _parse(self, entries, validate):
- """
- Parses a series of 'keyword => (value, pgp block)' mappings and applies
- them as attributes.
-
- :param dict entries: descriptor contents to be applied
- :param bool validate: checks the validity of descriptor content if **True**
-
- :raises: **ValueError** if an error occurs in validation
- """
-
- for keyword, values in list(entries.items()):
- # most just work with the first (and only) value
- value, block_type, block_contents = values[0]
-
- line = '%s %s' % (keyword, value) # original line
-
- if block_contents:
- line += '\n%s' % block_contents
-
- if keyword == 'onion-key':
- if validate and (not block_contents or block_type != 'RSA PUBLIC KEY'):
- raise ValueError("'onion-key' should be followed by a RSA PUBLIC KEY block: %s" % line)
-
- self.onion_key = block_contents
- elif keyword == 'ntor-onion-key':
- self.ntor_onion_key = value
- elif keyword == 'a':
- for entry, _, _ in values:
- stem.descriptor.router_status_entry._parse_a_line(self, entry, validate)
- elif keyword == 'family':
- self.family = value.split(' ')
- elif keyword == 'p':
- stem.descriptor.router_status_entry._parse_p_line(self, value, validate)
- elif keyword == 'p6':
- self.exit_policy_v6 = stem.exit_policy.MicroExitPolicy(value)
- elif keyword == 'id':
- value_comp = value.split()
-
- if len(value_comp) >= 2:
- self.identifier_type = value_comp[0]
- self.identifier = value_comp[1]
- elif validate:
- raise ValueError("'id' lines should contain both the key type and digest: %s" % line)
- else:
- self._unrecognized_lines.append(line)
-
def _check_constraints(self, entries):
"""
Does a basic check that the entries conform to this descriptor type's
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py
index b6af898..a696d6a 100644
--- a/stem/descriptor/server_descriptor.py
+++ b/stem/descriptor/server_descriptor.py
@@ -533,7 +533,7 @@ class ServerDescriptor(Descriptor):
:raises: **ValueError** if the contents is malformed and validate is True
"""
- super(ServerDescriptor, self).__init__(raw_contents)
+ super(ServerDescriptor, self).__init__(raw_contents, lazy_load = not validate)
# Only a few things can be arbitrary bytes according to the dir-spec, so
# parsing them separately.
@@ -541,9 +541,6 @@ class ServerDescriptor(Descriptor):
self.platform = _get_bytes_field('platform', raw_contents)
self.contact = _get_bytes_field('contact', raw_contents)
- raw_contents = stem.util.str_tools._to_unicode(raw_contents)
-
- self._lazy_loading = not validate
self._annotation_lines = annotations if annotations else []
# A descriptor contains a series of 'keyword lines' which are simply a
@@ -554,6 +551,7 @@ class ServerDescriptor(Descriptor):
# influences the resulting exit policy, but for everything else the order
# does not matter so breaking it into key / value pairs.
+ raw_contents = stem.util.str_tools._to_unicode(raw_contents)
entries, self._unparsed_exit_policy = _get_descriptor_components(raw_contents, validate, ('accept', 'reject'))
if validate:
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits