[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [stem/master] Moving DocumentHeader parsing to helpers



commit a1575e9ff1aa053fa19fde4d715ecd1f5a3ace0a
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date:   Sun Jan 18 13:00:12 2015 -0800

    Moving DocumentHeader parsing to helpers
    
    The v3 network status document is gonna be a bit trickier since it delegates
    parsing to sub-objects. Essentially it acts as a collection of sub-documents,
    then adds those attributes to itself.
    
    Starting by moving the header parsing to helpers like the other document types.
---
 stem/descriptor/networkstatus.py                  |  327 ++++++++++++---------
 test/unit/descriptor/networkstatus/document_v3.py |   30 +-
 2 files changed, 199 insertions(+), 158 deletions(-)

diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py
index 29d2593..910f684 100644
--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -569,7 +569,175 @@ class NetworkStatusDocumentV3(NetworkStatusDocument):
     return self._compare(other, lambda s, o: s <= o)
 
 
+def _parse_network_status_version_line(descriptor, entries):
+  # "network-status-version" version
+
+  value = _value('network-status-version', entries)
+
+  if ' ' in value:
+    version, flavor = value.split(' ', 1)
+  else:
+    version, flavor = value, None
+
+  if not version.isdigit():
+    raise ValueError('Network status document has a non-numeric version: network-status-version %s' % value)
+
+  descriptor.version = int(version)
+  descriptor.version_flavor = flavor
+  descriptor.is_microdescriptor = flavor == 'microdesc'
+
+  if descriptor.version != 3:
+    raise ValueError("Expected a version 3 network status document, got version '%s' instead" % descriptor.version)
+
+
+def _parse_vote_status_line(descriptor, entries):
+  # "vote-status" type
+  #
+  # The consensus-method and consensus-methods fields are optional since
+  # they weren't included in version 1. Setting a default now that we
+  # know if we're a vote or not.
+
+  value = _value('vote-status', entries)
+
+  if value == 'consensus':
+    descriptor.is_consensus, descriptor.is_vote = True, False
+  elif value == 'vote':
+    descriptor.is_consensus, descriptor.is_vote = False, True
+  else:
+    raise ValueError("A network status document's vote-status line can only be 'consensus' or 'vote', got '%s' instead" % value)
+
+
+def _parse_consensus_methods_line(descriptor, entries):
+  # "consensus-methods" IntegerList
+
+  value, consensus_methods = _value('consensus-methods', entries), []
+
+  for entry in value.split(' '):
+    if not entry.isdigit():
+      raise ValueError("A network status document's consensus-methods must be a list of integer values, but was '%s'" % value)
+
+    consensus_methods.append(int(entry))
+
+  descriptor.consensus_methods = consensus_methods
+
+
+def _parse_consensus_method_line(descriptor, entries):
+  # "consensus-method" Integer
+
+  value = _value('consensus-method', entries)
+
+  if not value.isdigit():
+    raise ValueError("A network status document's consensus-method must be an integer, but was '%s'" % value)
+
+  descriptor.consensus_method = int(value)
+
+
+def _parse_voting_delay_line(descriptor, entries):
+  # "voting-delay" VoteSeconds DistSeconds
+
+  value = _value('voting-delay', entries)
+  value_comp = value.split(' ')
+
+  if len(value_comp) == 2 and value_comp[0].isdigit() and value_comp[1].isdigit():
+    descriptor.vote_delay = int(value_comp[0])
+    descriptor.dist_delay = int(value_comp[1])
+  else:
+    raise ValueError("A network status document's 'voting-delay' line must be a pair of integer values, but was '%s'" % value)
+
+
+def _parse_versions_line(keyword, attribute):
+  def _parse(descriptor, entries):
+    value, entries = _value(keyword, entries), []
+
+    for entry in value.split(','):
+      try:
+        entries.append(stem.version._get_version(entry))
+      except ValueError:
+        raise ValueError("Network status document's '%s' line had '%s', which isn't a parsable tor version: %s %s" % (keyword, entry, keyword, value))
+
+    setattr(descriptor, attribute, entries)
+
+  return _parse
+
+
+def _parse_flag_thresholds_line(descriptor, entries):
+  # "flag-thresholds" SP THRESHOLDS
+
+  value, thresholds = _value('flag-thresholds', entries).strip(), {}
+
+  if value:
+    for entry in value.split(' '):
+      if '=' not in entry:
+        raise ValueError("Network status document's 'flag-thresholds' line is expected to be space separated key=value mappings, got: flag-thresholds %s" % value)
+
+      entry_key, entry_value = entry.split('=', 1)
+
+      try:
+        if entry_value.endswith('%'):
+          # opting for string manipulation rather than just
+          # 'float(entry_value) / 100' because floating point arithmetic
+          # will lose precision
+
+          thresholds[entry_key] = float('0.' + entry_value[:-1].replace('.', '', 1))
+        elif '.' in entry_value:
+          thresholds[entry_key] = float(entry_value)
+        else:
+          thresholds[entry_key] = int(entry_value)
+      except ValueError:
+        raise ValueError("Network status document's 'flag-thresholds' line is expected to have float values, got: flag-thresholds %s" % value)
+
+  descriptor.flag_thresholds = thresholds
+
+
+def _parse_parameters_line(descriptor, entries):
+  # "params" [Parameters]
+  # Parameter ::= Keyword '=' Int32
+  # Int32 ::= A decimal integer between -2147483648 and 2147483647.
+  # Parameters ::= Parameter | Parameters SP Parameter
+
+  value = _value('params', entries)
+
+  # should only appear in consensus-method 7 or later
+
+  if not descriptor.meets_consensus_method(7):
+    raise ValueError("A network status document's 'params' line should only appear in consensus-method 7 or later")
+
+  # skip if this is a blank line
+
+  params = dict(DEFAULT_PARAMS) if descriptor._default_params else {}
+
+  if value != '':
+    params.update(_parse_int_mappings('params', value, True))
+    descriptor.params = params
+    descriptor._check_params_constraints()
+
+
+_parse_valid_after_line = _parse_timestamp_line('valid-after', 'valid_after')
+_parse_fresh_until_line = _parse_timestamp_line('fresh-until', 'fresh_until')
+_parse_valid_until_line = _parse_timestamp_line('valid-until', 'valid_until')
+_parse_client_versions_line = _parse_versions_line('client-versions', 'client_versions')
+_parse_server_versions_line = _parse_versions_line('server-versions', 'server_versions')
+_parse_known_flags_line = lambda descriptor, entries: setattr(descriptor, 'known_flags', [entry for entry in _value('known-flags', entries).split(' ') if entry])
+
+
 class _DocumentHeader(object):
+  PARSER_FOR_LINE = {
+    'network-status-version': _parse_network_status_version_line,
+    'vote-status': _parse_vote_status_line,
+    'consensus-methods': _parse_consensus_methods_line,
+    'consensus-method': _parse_consensus_method_line,
+    'published': _parse_published_line,
+    'valid-after': _parse_valid_after_line,
+    'fresh-until': _parse_fresh_until_line,
+    'valid-until': _parse_valid_until_line,
+    'voting-delay': _parse_voting_delay_line,
+    'client-versions': _parse_client_versions_line,
+    'server-versions': _parse_server_versions_line,
+    'known-flags': _parse_known_flags_line,
+    'flag-thresholds': _parse_flag_thresholds_line,
+    'params': _parse_parameters_line,
+  }
+
   def __init__(self, document_file, validate, default_params):
     self.version = None
     self.version_flavor = None
@@ -590,6 +758,8 @@ class _DocumentHeader(object):
     self.flag_thresholds = {}
     self.params = dict(DEFAULT_PARAMS) if default_params else {}
 
+    self._default_params = default_params
+
     self._unrecognized_lines = []
 
     content = bytes.join(b'', _read_until_keywords((AUTH_START, ROUTERS_START, FOOTER_START), document_file))
@@ -621,152 +791,21 @@ class _DocumentHeader(object):
       if validate and len(values) > 1 and keyword in HEADER_FIELDS:
         raise ValueError("Network status documents can only have a single '%s' line, got %i" % (keyword, len(values)))
 
-      if keyword == 'network-status-version':
-        # "network-status-version" version
-
-        if ' ' in value:
-          version, flavor = value.split(' ', 1)
+      try:
+        if keyword in self.PARSER_FOR_LINE:
+          self.PARSER_FOR_LINE[keyword](self, entries)
         else:
-          version, flavor = value, None
-
-        if not version.isdigit():
-          if not validate:
-            continue
-
-          raise ValueError('Network status document has a non-numeric version: %s' % line)
-
-        self.version = int(version)
-        self.version_flavor = flavor
-        self.is_microdescriptor = flavor == 'microdesc'
-
-        if validate and self.version != 3:
-          raise ValueError("Expected a version 3 network status document, got version '%s' instead" % self.version)
-      elif keyword == 'vote-status':
-        # "vote-status" type
-        #
-        # The consensus-method and consensus-methods fields are optional since
-        # they weren't included in version 1. Setting a default now that we
-        # know if we're a vote or not.
-
-        if value == 'consensus':
-          self.is_consensus, self.is_vote = True, False
-          self.consensus_method = 1
-        elif value == 'vote':
-          self.is_consensus, self.is_vote = False, True
-          self.consensus_methods = [1]
-        elif validate:
-          raise ValueError("A network status document's vote-status line can only be 'consensus' or 'vote', got '%s' instead" % value)
-      elif keyword == 'consensus-methods':
-        # "consensus-methods" IntegerList
-
-        consensus_methods = []
-        for entry in value.split(' '):
-          if entry.isdigit():
-            consensus_methods.append(int(entry))
-          elif validate:
-            raise ValueError("A network status document's consensus-methods must be a list of integer values, but was '%s'" % value)
-
-        self.consensus_methods = consensus_methods
-      elif keyword == 'consensus-method':
-        # "consensus-method" Integer
-
-        if value.isdigit():
-          self.consensus_method = int(value)
-        elif validate:
-          raise ValueError("A network status document's consensus-method must be an integer, but was '%s'" % value)
-      elif keyword in ('published', 'valid-after', 'fresh-until', 'valid-until'):
-        try:
-          date_value = stem.util.str_tools._parse_timestamp(value)
-
-          if keyword == 'published':
-            self.published = date_value
-          elif keyword == 'valid-after':
-            self.valid_after = date_value
-          elif keyword == 'fresh-until':
-            self.fresh_until = date_value
-          elif keyword == 'valid-until':
-            self.valid_until = date_value
-        except ValueError:
-          if validate:
-            raise ValueError("Network status document's '%s' time wasn't parsable: %s" % (keyword, value))
-      elif keyword == 'voting-delay':
-        # "voting-delay" VoteSeconds DistSeconds
-
-        value_comp = value.split(' ')
-
-        if len(value_comp) == 2 and value_comp[0].isdigit() and value_comp[1].isdigit():
-          self.vote_delay = int(value_comp[0])
-          self.dist_delay = int(value_comp[1])
-        elif validate:
-          raise ValueError("A network status document's 'voting-delay' line must be a pair of integer values, but was '%s'" % value)
-      elif keyword in ('client-versions', 'server-versions'):
-        for entry in value.split(','):
-          try:
-            version_value = stem.version._get_version(entry)
-
-            if keyword == 'client-versions':
-              self.client_versions.append(version_value)
-            elif keyword == 'server-versions':
-              self.server_versions.append(version_value)
-          except ValueError:
-            if validate:
-              raise ValueError("Network status document's '%s' line had '%s', which isn't a parsable tor version: %s" % (keyword, entry, line))
-      elif keyword == 'known-flags':
-        # "known-flags" FlagList
-
-        # simply fetches the entries, excluding empty strings
-        self.known_flags = [entry for entry in value.split(' ') if entry]
-      elif keyword == 'flag-thresholds':
-        # "flag-thresholds" SP THRESHOLDS
-
-        value = value.strip()
-
-        if value:
-          for entry in value.split(' '):
-            if '=' not in entry:
-              if not validate:
-                continue
-
-              raise ValueError("Network status document's '%s' line is expected to be space separated key=value mappings, got: %s" % (keyword, line))
-
-            entry_key, entry_value = entry.split('=', 1)
-
-            try:
-              if entry_value.endswith('%'):
-                # opting for string manipulation rather than just
-                # 'float(entry_value) / 100' because floating point arithmetic
-                # will lose precision
-
-                self.flag_thresholds[entry_key] = float('0.' + entry_value[:-1].replace('.', '', 1))
-              elif '.' in entry_value:
-                self.flag_thresholds[entry_key] = float(entry_value)
-              else:
-                self.flag_thresholds[entry_key] = int(entry_value)
-            except ValueError:
-              if validate:
-                raise ValueError("Network status document's '%s' line is expected to have float values, got: %s" % (keyword, line))
-      elif keyword == 'params':
-        # "params" [Parameters]
-        # Parameter ::= Keyword '=' Int32
-        # Int32 ::= A decimal integer between -2147483648 and 2147483647.
-        # Parameters ::= Parameter | Parameters SP Parameter
-
-        # should only appear in consensus-method 7 or later
-
-        if validate and not self.meets_consensus_method(7):
-          raise ValueError("A network status document's 'params' line should only appear in consensus-method 7 or later")
-
-        # skip if this is a blank line
-
-        if value == '':
-          continue
-
-        self.params.update(_parse_int_mappings(keyword, value, validate))
-
+          self._unrecognized_lines.append(line)
+      except ValueError as exc:
         if validate:
-          self._check_params_constraints()
-      else:
-        self._unrecognized_lines.append(line)
+          raise exc
+
+    # default consensus_method and consensus_methods based on if we're a consensus or vote
+
+    if self.is_consensus and not self.consensus_method:
+      self.consensus_method = 1
+    elif self.is_vote and not self.consensus_methods:
+      self.consensus_methods = [1]
 
   def _check_params_constraints(self):
     """
diff --git a/test/unit/descriptor/networkstatus/document_v3.py b/test/unit/descriptor/networkstatus/document_v3.py
index 4b8efc4..e31600f 100644
--- a/test/unit/descriptor/networkstatus/document_v3.py
+++ b/test/unit/descriptor/networkstatus/document_v3.py
@@ -586,19 +586,21 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w=
     self.assertEqual(None, document.consensus_method)
 
     test_values = (
-      ('', []),
-      ('   ', []),
-      ('1 2 3 a 5', [1, 2, 3, 5]),
-      ('1 2 3 4.0 5', [1, 2, 3, 5]),
-      ('2 3 4', [2, 3, 4]),  # spec says version one must be included
+      (''),
+      ('   '),
+      ('1 2 3 a 5'),
+      ('1 2 3 4.0 5'),
+      ('2 3 4'),  # spec says version one must be included
     )
 
-    for test_value, expected_consensus_methods in test_values:
+    for test_value in test_values:
       content = get_network_status_document_v3({'vote-status': 'vote', 'consensus-methods': test_value}, content = True)
       self.assertRaises(ValueError, NetworkStatusDocumentV3, content)
 
+      expected_value = [2, 3, 4] if test_value == '2 3 4' else [1]
+
       document = NetworkStatusDocumentV3(content, False)
-      self.assertEqual(expected_consensus_methods, document.consensus_methods)
+      self.assertEqual(expected_value, document.consensus_methods)
 
   def test_consensus_method(self):
     """
@@ -708,21 +710,21 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w=
     self.assertEqual(expected, document.server_versions)
 
     test_values = (
-      ('', []),
-      ('   ', []),
-      ('1.2.3.4,', [stem.version.Version('1.2.3.4')]),
-      ('1.2.3.4,1.2.3.a', [stem.version.Version('1.2.3.4')]),
+      (''),
+      ('   '),
+      ('1.2.3.4,'),
+      ('1.2.3.4,1.2.3.a'),
     )
 
     for field in ('client-versions', 'server-versions'):
       attr = field.replace('-', '_')
 
-      for test_value, expected_value in test_values:
+      for test_value in test_values:
         content = get_network_status_document_v3({field: test_value}, content = True)
         self.assertRaises(ValueError, NetworkStatusDocumentV3, content)
 
         document = NetworkStatusDocumentV3(content, False)
-        self.assertEqual(expected_value, getattr(document, attr))
+        self.assertEqual([], getattr(document, attr))
 
   def test_known_flags(self):
     """
@@ -872,7 +874,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w=
     self.assertRaises(ValueError, NetworkStatusDocumentV3, content)
 
     document = NetworkStatusDocumentV3(content, False, default_params = False)
-    self.assertEqual({'unrecognized': -122, 'bwauthpid': 1}, document.params)
+    self.assertEqual({}, document.params)
 
   def test_footer_consensus_method_requirement(self):
     """



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits