[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [stem/master] Supporting microdescriptor flavored consensuses



commit 4216b5f1d5762d229945306508ea078c9fd1902c
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date:   Sun Oct 7 18:47:27 2012 -0700

    Supporting microdescriptor flavored consensuses
    
    Adding support for microdescriptor flavored consensuses into the
    NetworkStatusDocument class. It made sense to have a separate class for it, but
    on the other hand it *is* still a v3 consensus and the only impact the flavor
    has is alternate router status entries so just blending a 'flavor' and
    'is_microdescriptor' attribute in.
---
 stem/descriptor/networkstatus.py               |  208 ++++--------------------
 test/unit/descriptor/networkstatus/document.py |   72 ++++++++-
 2 files changed, 100 insertions(+), 180 deletions(-)

diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py
index 38207b8..d9aab4f 100644
--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -156,11 +156,9 @@ def parse_file(document_file, validate = True, is_microdescriptor = False):
   document_content = "".join(header + footer)
   
   if not is_microdescriptor:
-    document = NetworkStatusDocument(document_content, validate)
     router_type = stem.descriptor.router_status_entry.RouterStatusEntryV3
   else:
-    document = MicrodescriptorConsensus(document_content, validate)
-    router_type = RouterMicrodescriptor
+    router_type = stem.descriptor.router_status_entry.RouterStatusEntryMicroV3
   
   desc_iterator = _get_entries(
     document_file,
@@ -169,7 +167,7 @@ def parse_file(document_file, validate = True, is_microdescriptor = False):
     entry_keyword = ROUTERS_START,
     start_position = routers_start,
     end_position = routers_end,
-    extra_args = (document,),
+    extra_args = (NetworkStatusDocument(document_content, validate),),
   )
   
   for desc in desc_iterator:
@@ -221,9 +219,11 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):
   
   :var tuple routers: RouterStatusEntryV3 contained in the document
   
-  :var str version: **\*** document version
+  :var int version: **\*** document version
+  :var str version_flavor: **\*** flavor associated with the document (such as 'microdesc')
   :var bool is_consensus: **\*** true if the document is a consensus
   :var bool is_vote: **\*** true if the document is a vote
+  :var bool is_microdescriptor: **\*** true if this is a microdescriptor flavored document, false otherwise
   :var datetime valid_after: **\*** time when the consensus became valid
   :var datetime fresh_until: **\*** time when the next consensus should be produced
   :var datetime valid_until: **\*** time when this consensus becomes obsolete
@@ -262,6 +262,14 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):
     document_file = StringIO(raw_content)
     
     self._header = _DocumentHeader(document_file, validate, default_params)
+    self._unrecognized_lines = []
+    
+    # merge header attributes into us
+    for attr, value in vars(self._header).items():
+      if attr != "_unrecognized_lines":
+        setattr(self, attr, value)
+      else:
+        self._unrecognized_lines += value
     
     self.directory_authorities = tuple(_get_entries(
       document_file,
@@ -272,28 +280,29 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):
       extra_args = (self._header.is_vote,),
     ))
     
+    if not self._header.is_microdescriptor:
+      router_type = stem.descriptor.router_status_entry.RouterStatusEntryV3
+    else:
+      router_type = stem.descriptor.router_status_entry.RouterStatusEntryMicroV3
+    
     self.routers = tuple(_get_entries(
       document_file,
       validate,
-      entry_class = self._get_router_type(),
+      entry_class = router_type,
       entry_keyword = ROUTERS_START,
       section_end_keywords = FOOTER_START,
       extra_args = (self,),
     ))
     
     self._footer = _DocumentFooter(document_file, validate, self._header)
-    self._unrecognized_lines = []
     
-    # copy the header and footer attributes into us
-    for attr, value in vars(self._header).items() + vars(self._footer).items():
+    # merge header attributes into us
+    for attr, value in vars(self._footer).items():
       if attr != "_unrecognized_lines":
         setattr(self, attr, value)
       else:
         self._unrecognized_lines += value
   
-  def _get_router_type(self):
-    return stem.descriptor.router_status_entry.RouterStatusEntryV3
-  
   def meets_consensus_method(self, method):
     """
     Checks if we meet the given consensus-method. This works for both votes and
@@ -319,8 +328,10 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):
 class _DocumentHeader(object):
   def __init__(self, document_file, validate, default_params):
     self.version = None
+    self.version_flavor = None
     self.is_consensus = True
     self.is_vote = False
+    self.is_microdescriptor = False
     self.consensus_methods = []
     self.published = None
     self.consensus_method = None
@@ -362,12 +373,20 @@ class _DocumentHeader(object):
       if keyword == 'network-status-version':
         # "network-status-version" version
         
-        self.version = value
+        if ' ' in value:
+          version, flavor = value.split(' ', 1)
+        else:
+          version, flavor = value, None
+        
+        if not version.isdigit():
+          if not validate: continue
+          raise ValueError("Network status document has a non-numeric version: %s" % line)
         
-        # TODO: Obviously not right when we extend this to parse v2 documents,
-        # but we'll cross that bridge when we come to it.
+        self.version = int(version)
+        self.version_flavor = flavor
+        self.is_microdescriptor = flavor == 'microdesc'
         
-        if validate and self.version != "3":
+        if validate and self.version != 3:
           raise ValueError("Expected a version 3 network status documents, got version '%s' instead" % self.version)
       elif keyword == 'vote-status':
         # "vote-status" type
@@ -1034,160 +1053,3 @@ class DocumentSignature(object):
     
     return 0
 
-class MicrodescriptorConsensus(NetworkStatusDocument):
-  """
-  A v3 microdescriptor consensus.
-  
-  :var str version: **\*** a document format version. For v3 microdescriptor consensuses this is "3 microdesc"
-  :var bool is_consensus: **\*** true if the document is a consensus
-  :var bool is_vote: **\*** true if the document is a vote
-  :var int consensus_method: **~** consensus method used to generate a consensus
-  :var datetime valid_after: **\*** time when the consensus becomes valid
-  :var datetime fresh_until: **\*** time until when the consensus is considered to be fresh
-  :var datetime valid_until: **\*** time until when the consensus is valid
-  :var int vote_delay: **\*** number of seconds allowed for collecting votes from all authorities
-  :var int dist_delay: number of seconds allowed for collecting signatures from all authorities
-  :var list client_versions: list of recommended Tor client versions
-  :var list server_versions: list of recommended Tor server versions
-  :var list known_flags: **\*** list of known router flags
-  :var list params: dict of parameter(str) => value(int) mappings
-  :var list directory_authorities: **\*** list of DirectoryAuthority objects that have generated this document
-  :var dict bandwidth_weights: **~** dict of weight(str) => value(int) mappings
-  :var list signatures: **\*** list of signatures this document has
-  
-  | **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined
-  | **~** attribute appears only in consensuses
-  """
-  
-  def _get_router_type(self):
-    return RouterMicrodescriptor
-  
-  def _validate_network_status_version(self):
-    return self.version == "3 microdesc"
-
-class RouterMicrodescriptor(stem.descriptor.router_status_entry.RouterStatusEntry):
-  """
-  Router microdescriptor object. Parses and stores router information in a router
-  microdescriptor from a v3 microdescriptor consensus.
-  
-  :var MicrodescriptorConsensus document: **\*** document this descriptor came from
-  
-  :var str nickname: **\*** router's nickname
-  :var str fingerprint: **\*** router's fingerprint
-  :var datetime published: **\*** router's publication
-  :var str ip: **\*** router's IP address
-  :var int or_port: **\*** router's ORPort
-  :var int dir_port: **\*** router's DirPort
-  
-  :var list flags: **\*** list of status flags
-  
-  :var :class:`stem.version.Version`,str version: Version of the Tor protocol this router is running
-  
-  :var int bandwidth: router's claimed bandwidth
-  :var int measured_bandwidth: router's measured bandwidth
-  
-  :var str digest: base64 of the hash of the router's microdescriptor with trailing =s omitted
-  
-  | **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined
-  """
-  
-  def __init__(self, raw_contents, validate, document):
-    """
-    Parse a router descriptor in a v3 microdescriptor consensus and provide a new
-    RouterMicrodescriptor object.
-    
-    :param str raw_content: router descriptor content to be parsed
-    :param MicrodescriptorConsensus document: document this descriptor came from
-    :param bool validate: whether the router descriptor should be validated
-    
-    :raises: ValueError if the descriptor data is invalid
-    """
-    
-    super(RouterMicrodescriptor, self).__init__(raw_contents, validate, document)
-    
-    self.document = document
-  
-  def _parse(self, raw_content, validate):
-    """
-    :param dict raw_content: router descriptor contents to be parsed
-    :param bool validate: checks the validity of descriptor content if True
-    
-    :raises: ValueError if an error occures in validation
-    """
-    
-    content = StringIO(raw_content)
-    seen_keywords = set()
-    peek_check_kw = lambda keyword: keyword == _peek_keyword(content)
-    
-    r = _read_keyword_line("r", content, validate)
-    # r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0
-    # "r" SP nickname SP identity SP digest SP publication SP IP SP ORPort SP DirPort NL
-    if r:
-      seen_keywords.add("r")
-      values = r.split(" ")
-      self.nickname, self.fingerprint = values[0], _decode_fingerprint(values[1], validate)
-      self.published = _strptime(" ".join((values[2], values[3])), validate)
-      self.ip, self.or_port, self.dir_port = values[4], int(values[5]), int(values[6])
-      if self.dir_port == 0: self.dir_port = None
-    elif validate: raise ValueError("Invalid router descriptor: empty 'r' line")
-    
-    while _peek_line(content):
-      if peek_check_kw("s"):
-        if "s" in seen_keywords: raise ValueError("Invalid router descriptor: 's' line appears twice")
-        line = _read_keyword_line("s", content, validate)
-        if not line: continue
-        seen_keywords.add("s")
-        # s Named Running Stable Valid
-        #A series of space-separated status flags, in *lexical order*
-        self.flags = line.split(" ")
-      
-      elif peek_check_kw("v"):
-        if "v" in seen_keywords: raise ValueError("Invalid router descriptor: 'v' line appears twice")
-        line = _read_keyword_line("v", content, validate, True)
-        seen_keywords.add("v")
-        # v Tor 0.2.2.35
-        if line:
-          if line.startswith("Tor "):
-            self.version = stem.version.Version(line[4:])
-          else:
-            self.version = line
-        elif validate: raise ValueError("Invalid router descriptor: empty 'v' line" )
-      
-      elif peek_check_kw("w"):
-        if "w" in seen_keywords: raise ValueError("Invalid router descriptor: 'w' line appears twice")
-        w = _read_keyword_line("w", content, validate, True)
-        # "w" SP "Bandwidth=" INT [SP "Measured=" INT] NL
-        seen_keywords.add("w")
-        if w:
-          values = w.split(" ")
-          if len(values) <= 2 and len(values) > 0:
-            key, value = values[0].split("=")
-            if key == "Bandwidth": self.bandwidth = int(value)
-            elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Bandwidth, read " + key)
-            
-            if len(values) == 2:
-              key, value = values[1].split("=")
-              if key == "Measured": self.measured_bandwidth = int(value)
-              elif validate: raise ValueError("Router descriptor contains invalid 'w' line: expected Measured, read " + key)
-          elif validate: raise ValueError("Router descriptor contains invalid 'w' line")
-        elif validate: raise ValueError("Router descriptor contains empty 'w' line")
-      
-      elif peek_check_kw("m"):
-        # microdescriptor hashes
-        self.digest = _read_keyword_line("m", content, validate, True)
-      
-      elif validate:
-        raise ValueError("Router descriptor contains unrecognized trailing lines: %s" % content.readline())
-      
-      else:
-        self.unrecognized_lines.append(content.readline()) # ignore unrecognized lines if we aren't validating
-  
-  def get_unrecognized_lines(self):
-    """
-    Returns any unrecognized lines.
-    
-    :returns: a list of unrecognized lines
-    """
-    
-    return self.unrecognized_lines
-
diff --git a/test/unit/descriptor/networkstatus/document.py b/test/unit/descriptor/networkstatus/document.py
index e92aee2..8009fbb 100644
--- a/test/unit/descriptor/networkstatus/document.py
+++ b/test/unit/descriptor/networkstatus/document.py
@@ -9,8 +9,8 @@ import StringIO
 import stem.version
 from stem.descriptor import Flag
 from stem.descriptor.networkstatus import HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS, DEFAULT_PARAMS, BANDWIDTH_WEIGHT_ENTRIES, DirectoryAuthority, NetworkStatusDocument, parse_file
-from stem.descriptor.router_status_entry import RouterStatusEntryV3
-from test.mocking import get_router_status_entry_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG
+from stem.descriptor.router_status_entry import RouterStatusEntryV3, RouterStatusEntryMicroV3
+from test.mocking import get_router_status_entry_v3, get_router_status_entry_micro_v3, get_directory_authority, get_network_status_document, CRYPTO_BLOB, DOC_SIG
 
 class TestNetworkStatusDocument(unittest.TestCase):
   def test_minimal_consensus(self):
@@ -25,9 +25,11 @@ class TestNetworkStatusDocument(unittest.TestCase):
       Flag.STABLE, Flag.UNNAMED, Flag.V2DIR, Flag.VALID]
     
     self.assertEqual((), document.routers)
-    self.assertEqual("3", document.version)
+    self.assertEqual(3, document.version)
+    self.assertEqual(None, document.version_flavor)
     self.assertEqual(True, document.is_consensus)
     self.assertEqual(False, document.is_vote)
+    self.assertEqual(False, document.is_microdescriptor)
     self.assertEqual(9, document.consensus_method)
     self.assertEqual([], document.consensus_methods)
     self.assertEqual(None, document.published)
@@ -57,7 +59,7 @@ class TestNetworkStatusDocument(unittest.TestCase):
       Flag.STABLE, Flag.UNNAMED, Flag.V2DIR, Flag.VALID]
     
     self.assertEqual((), document.routers)
-    self.assertEqual("3", document.version)
+    self.assertEqual(3, document.version)
     self.assertEqual(False, document.is_consensus)
     self.assertEqual(True, document.is_vote)
     self.assertEqual(None, document.consensus_method)
@@ -178,13 +180,22 @@ class TestNetworkStatusDocument(unittest.TestCase):
     """
     
     document = get_network_status_document({"network-status-version": "3"})
-    self.assertEquals("3", document.version)
+    self.assertEquals(3, document.version)
+    self.assertEquals(None, document.version_flavor)
+    self.assertEquals(False, document.is_microdescriptor)
+    
+    document = get_network_status_document({"network-status-version": "3 microdesc"})
+    self.assertEquals(3, document.version)
+    self.assertEquals('microdesc', document.version_flavor)
+    self.assertEquals(True, document.is_microdescriptor)
     
     content = get_network_status_document({"network-status-version": "4"}, content = True)
     self.assertRaises(ValueError, NetworkStatusDocument, content)
     
     document = NetworkStatusDocument(content, False)
-    self.assertEquals("4", document.version)
+    self.assertEquals(4, document.version)
+    self.assertEquals(None, document.version_flavor)
+    self.assertEquals(False, document.is_microdescriptor)
   
   def test_vote_status(self):
     """
@@ -616,7 +627,7 @@ class TestNetworkStatusDocument(unittest.TestCase):
   
   def test_with_router_status_entries(self):
     """
-    Includes a router status entry within the document. This isn't to test the
+    Includes router status entries within the document. This isn't to test the
     RouterStatusEntry parsing but rather the inclusion of it within the
     document.
     """
@@ -635,6 +646,53 @@ class TestNetworkStatusDocument(unittest.TestCase):
     self.assertRaises(ValueError, NetworkStatusDocument, content)
     document = NetworkStatusDocument(content, False)
     self.assertEquals((entry3,), document.routers)
+    
+    # try including with a microdescriptor consensus
+    
+    content = get_network_status_document({"network-status-version": "3 microdesc"}, routers = (entry1, entry2), content = True)
+    self.assertRaises(ValueError, NetworkStatusDocument, content)
+    
+    expected_routers = (
+      RouterStatusEntryMicroV3(str(entry1), False),
+      RouterStatusEntryMicroV3(str(entry2), False),
+    )
+    
+    document = NetworkStatusDocument(content, False)
+    self.assertEquals(expected_routers, document.routers)
+  
+  def test_with_microdescriptor_router_status_entries(self):
+    """
+    Includes microdescriptor flavored router status entries within the
+    document.
+    """
+    
+    entry1 = get_router_status_entry_micro_v3({'s': "Fast"})
+    entry2 = get_router_status_entry_micro_v3({'s': "Valid"})
+    document = get_network_status_document({"network-status-version": "3 microdesc"}, routers = (entry1, entry2))
+    
+    self.assertEquals((entry1, entry2), document.routers)
+    
+    # try with an invalid RouterStatusEntry
+    
+    entry3 = RouterStatusEntryMicroV3(get_router_status_entry_micro_v3({'r': "ugabuga"}, content = True), False)
+    content = get_network_status_document({"network-status-version": "3 microdesc"}, routers = (entry3,), content = True)
+    
+    self.assertRaises(ValueError, NetworkStatusDocument, content)
+    document = NetworkStatusDocument(content, False)
+    self.assertEquals((entry3,), document.routers)
+    
+    # try including microdescriptor entries in a normal consensus
+    
+    content = get_network_status_document(routers = (entry1, entry2), content = True)
+    self.assertRaises(ValueError, NetworkStatusDocument, content)
+    
+    expected_routers = (
+      RouterStatusEntryV3(str(entry1), False),
+      RouterStatusEntryV3(str(entry2), False),
+    )
+    
+    document = NetworkStatusDocument(content, False)
+    self.assertEquals(expected_routers, document.routers)
   
   def test_with_directory_authorities(self):
     """



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits