[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [stem/master] Cleaning up prior descriptor implementations



commit 7e9d454ed4eafe6b5461c5ff5170ca5634899edf
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date:   Sat Mar 24 13:41:11 2012 -0700

    Cleaning up prior descriptor implementations
    
    Adding header pydocs and made a variety of small fixes and naming improvements.
---
 stem/descriptor/__init__.py          |   45 +++++++----------
 stem/descriptor/reader.py            |    4 +-
 stem/descriptor/server_descriptor.py |   86 +++++++++++++++++++++++-----------
 test/integ/descriptor/reader.py      |    4 +-
 4 files changed, 81 insertions(+), 58 deletions(-)

diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index ad17296..215864f 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -1,12 +1,18 @@
 """
 Package for parsing and processing descriptor data.
+
+parse_file - Iterates over the descriptors in a file.
+Descriptor - Common parent for all descriptor file types.
+  |- get_path - location of the descriptor on disk if it came from a file
+  |- get_unrecognized_lines - unparsed descriptor content
+  +- __str__ - string that the descriptor was made from
 """
 
-__all__ = ["descriptor", "reader", "server_descriptor", "parse_descriptors", "Descriptor"]
+__all__ = ["descriptor", "reader", "server_descriptor", "parse_file", "Descriptor"]
 
 import os
 
-def parse_descriptors(path, descriptor_file):
+def parse_file(path, descriptor_file):
   """
   Provides an iterator for the descriptors within a given file.
   
@@ -15,7 +21,7 @@ def parse_descriptors(path, descriptor_file):
     descriptor_file (file) - opened file with the descriptor contents
   
   Returns:
-    iterator that parses the file's contents into descriptors
+    iterator for Descriptor instances in the file
   
   Raises:
     TypeError if we can't match the contents of the file to a descriptor type
@@ -26,33 +32,21 @@ def parse_descriptors(path, descriptor_file):
   
   # The tor descriptor specifications do not provide a reliable method for
   # identifying a descriptor file's type and version so we need to guess
-  # based on its filename for resources from the data directory and contents
-  # for files provided by metrics.
+  # based on...
+  # - its filename for resources from the tor data directory
+  # - first line of our contents for files provided by metrics
   
   filename = os.path.basename(path)
-  
-  if filename == "cached-descriptors":
-    # server descriptors from tor's data directory
-    while descriptor_file:
-      yield stem.descriptor.server_descriptor.parse_server_descriptors_v2(path, descriptor_file)
-    
-    return
-  
   first_line = descriptor_file.readline()
   descriptor_file.seek(0)
   
-  if first_line.startswith("router "):
-    # server descriptor
-    for desc in stem.descriptor.server_descriptor.parse_server_descriptors_v2(path, descriptor_file):
+  if filename == "cached-descriptors" or first_line.startswith("router "):
+    for desc in stem.descriptor.server_descriptor.parse_file_v2(descriptor_file):
+      desc._set_path(path)
       yield desc
-    
-    return
-  
-  # TODO: implement actual descriptor type recognition and parsing
-  # TODO: add integ test for non-descriptor text content
-  desc = Descriptor(descriptor_file.read())
-  desc._set_path(path)
-  yield desc
+  else:
+    # unrecognized descritor type
+    raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line))
 
 class Descriptor:
   """
@@ -83,7 +77,7 @@ class Descriptor:
       list of lines of unrecognized content
     """
     
-    return []
+    raise NotImplementedError
   
   def _set_path(self, path):
     self._path = path
@@ -91,4 +85,3 @@ class Descriptor:
   def __str__(self):
     return self._raw_contents
 
-
diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py
index 4f043bc..511ba04 100644
--- a/stem/descriptor/reader.py
+++ b/stem/descriptor/reader.py
@@ -365,7 +365,7 @@ class DescriptorReader:
   def _handle_descriptor_file(self, target):
     try:
       with open(target) as target_file:
-        for desc in stem.descriptor.parse_descriptors(target, target_file):
+        for desc in stem.descriptor.parse_file(target, target_file):
           self._enqueue_descriptor(desc)
           self._iter_notice.set()
     except TypeError, exc:
@@ -380,7 +380,7 @@ class DescriptorReader:
           if tar_entry.isfile():
             entry = tar_file.extractfile(tar_entry)
             
-            for desc in stem.descriptor.parse_descriptors(target, entry):
+            for desc in stem.descriptor.parse_file(target, entry):
               self._enqueue_descriptor(desc)
               self._iter_notice.set()
             
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py
index fe6889b..70db1fe 100644
--- a/stem/descriptor/server_descriptor.py
+++ b/stem/descriptor/server_descriptor.py
@@ -6,15 +6,22 @@ etc). This information is provided from a few sources...
 - control port via 'GETINFO desc/*' queries
 - the 'cached-descriptors' file in tor's data directory
 - tor metrics, at https://metrics.torproject.org/data.html
+
+parse_file_v2 - Iterates over the server descriptors in a file.
+ServerDescriptorV2 - Tor server descriptor, version 2.
+  |- get_unrecognized_lines - lines with unrecognized content
+  |- get_annotations - dictionary of content prior to the descriptor entry
+  |- get_annotation_lines - lines that provided the annotations
+  +- is_valid - checks the signature against the descriptor content
 """
 
 import re
 import datetime
 
+import stem.descriptor
 import stem.version
 import stem.util.connection
 import stem.util.tor_tools
-import stem.descriptor
 
 ENTRY_START = "router"
 ENTRY_END   = "router-signature"
@@ -48,9 +55,21 @@ SINGLE_FIELDS = (
   "family",
 )
 
-def parse_server_descriptors_v2(path, descriptor_file):
+def parse_file_v2(descriptor_file, validate = True):
   """
-  Iterates over the verion 2 server descriptors in a descriptor file.
+  Iterates over the version 2 server descriptors in a file.
+  
+  Arguments:
+    descriptor_file (file) - file with descriptor content
+    validate (bool)        - checks the validity of the descriptor's content if
+                             True, skips these checks otherwise
+  
+  Returns:
+    iterator for ServerDescriptorV2 instances in the file
+  
+  Raises:
+    ValueError if the contents is malformed and validate is True
+    IOError if the file can't be read
   """
   
   # Cached descriptors consist of annotations followed by the descriptor
@@ -73,6 +92,9 @@ def parse_server_descriptors_v2(path, descriptor_file):
   #   - parse as descriptor content until we get to ENTRY_END followed by the
   #     end of the signature block
   #   - construct a descriptor and provide it back to the caller
+  #
+  # Any annotations after the last server descriptor is ignored (never provided
+  # to the caller).
   
   while True:
     annotations = _read_until_keyword(ENTRY_START, descriptor_file)
@@ -82,16 +104,14 @@ def parse_server_descriptors_v2(path, descriptor_file):
     block_end_prefix = PGP_BLOCK_END.split(' ', 1)[0]
     descriptor_content += _read_until_keyword(block_end_prefix, descriptor_file, True)
     
-    # If the file has ending annotations (ie, non-descriptor text after the
-    # last descriptor) then we won't have any descriptor content at this point.
-    # This is fine. Those ending annotations are simply never returned to the
-    # caller.
-    
     if descriptor_content:
-      descriptor = ServerDescriptorV2("\n".join(descriptor_content), annotations = annotations)
-      descriptor._set_path(path)
+      # strip newlines from annotations
+      annotations = map(str.strip, annotations)
+      
+      descriptor_text = "".join(descriptor_content)
+      descriptor = ServerDescriptorV2(descriptor_text, validate, annotations)
       yield descriptor
-    else: return # done parsing descriptors
+    else: break # done parsing descriptors
 
 def _read_until_keyword(keyword, descriptor_file, inclusive = False):
   """
@@ -112,12 +132,10 @@ def _read_until_keyword(keyword, descriptor_file, inclusive = False):
   while True:
     last_position = descriptor_file.tell()
     line = descriptor_file.readline()
-    
     if not line: break # EOF
-    line = line.strip()
     
     if " " in line: line_keyword = line.split(" ", 1)[0]
-    else: line_keyword = line
+    else: line_keyword = line.strip()
     
     if line_keyword == keyword:
       if inclusive: content.append(line)
@@ -157,13 +175,13 @@ def _get_psudo_pgp_block(remaining_contents):
     
     while True:
       if not remaining_contents:
-        raise ValueError("Unterminated public key block")
+        raise ValueError("Unterminated pgp style block")
       
       line = remaining_contents.pop(0)
       block_lines.append(line)
       
       if line == PGP_BLOCK_END % block_type:
-        return block_type, "\n".join(block_lines)
+        return (block_type, "\n".join(block_lines))
   else:
     return (None, None)
 
@@ -197,7 +215,7 @@ class ServerDescriptorV2(stem.descriptor.Descriptor):
     contact (str)            - relay's contact information
     family (list)            - nicknames or fingerprints of relays it has a declared family with (*)
     
-    * required fields, others are left as None if undefined
+    (*) required fields, others are left as None if undefined
   """
   
   nickname = address = or_port = socks_port = dir_port = None
@@ -206,7 +224,7 @@ class ServerDescriptorV2(stem.descriptor.Descriptor):
   onion_key = onion_key_type = signing_key = signing_key_type = None
   router_sig = router_sig_type = contact = None
   hibernating = False
-  family = unrecognized_entries = []
+  family = unrecognized_lines = []
   
   # TODO: Until we have a proper ExitPolicy class this is just a list of the
   # exit policy strings...
@@ -258,6 +276,9 @@ class ServerDescriptorV2(stem.descriptor.Descriptor):
     while remaining_contents:
       line = remaining_contents.pop(0)
       
+      # last line can be empty
+      if not line and not remaining_contents: continue
+      
       # Some lines have an 'opt ' for backward compatability. They should be
       # ignored. This prefix is being removed in...
       # https://trac.torproject.org/projects/tor/ticket/5124
@@ -271,7 +292,12 @@ class ServerDescriptorV2(stem.descriptor.Descriptor):
         raise ValueError("Line contains invalid characters: %s" % line)
       
       keyword, value = line_match.groups()
-      block_type, block_contents = _get_psudo_pgp_block(remaining_contents)
+      
+      try:
+        block_type, block_contents = _get_psudo_pgp_block(remaining_contents)
+      except ValueError, exc:
+        if not validate: continue
+        raise exc
       
       if keyword in ("accept", "reject"):
         self.exit_policy.append("%s %s" % (keyword, value))
@@ -291,7 +317,6 @@ class ServerDescriptorV2(stem.descriptor.Descriptor):
           raise ValueError("The '%s' entry can only appear once in a descriptor" % keyword)
     
     # parse all the entries into our attributes
-    
     for keyword, values in entries.items():
       # most just work with the first (and only) value
       value, block_type, block_contents = values[0]
@@ -332,13 +357,15 @@ class ServerDescriptorV2(stem.descriptor.Descriptor):
           if not validate: continue
           raise ValueError("Bandwidth line must have three values: %s" % line)
         
-        if validate:
-          if not bandwidth_comp[0].isdigit():
-            raise ValueError("Bandwidth line's average rate isn't numeric: %s" % bandwidth_comp[0])
-          elif not bandwidth_comp[1].isdigit():
-            raise ValueError("Bandwidth line's burst rate isn't numeric: %s" % bandwidth_comp[1])
-          elif not bandwidth_comp[2].isdigit():
-            raise ValueError("Bandwidth line's observed rate isn't numeric: %s" % bandwidth_comp[2])
+        if not bandwidth_comp[0].isdigit():
+          if not validate: continue
+          raise ValueError("Bandwidth line's average rate isn't numeric: %s" % bandwidth_comp[0])
+        elif not bandwidth_comp[1].isdigit():
+          if not validate: continue
+          raise ValueError("Bandwidth line's burst rate isn't numeric: %s" % bandwidth_comp[1])
+        elif not bandwidth_comp[2].isdigit():
+          if not validate: continue
+          raise ValueError("Bandwidth line's observed rate isn't numeric: %s" % bandwidth_comp[2])
         
         self.average_bandwidth  = int(bandwidth_comp[0])
         self.burst_bandwidth    = int(bandwidth_comp[1])
@@ -419,7 +446,10 @@ class ServerDescriptorV2(stem.descriptor.Descriptor):
       elif keyword == "family":
         self.family = value.split(" ")
       else:
-        self.unrecognized_entries.append(line)
+        self.unrecognized_lines.append(line)
+  
+  def get_unrecognized_lines(self):
+    return list(unrecognized_lines)
   
   def get_annotations(self):
     """
diff --git a/test/integ/descriptor/reader.py b/test/integ/descriptor/reader.py
index 7f2a425..cdc33c2 100644
--- a/test/integ/descriptor/reader.py
+++ b/test/integ/descriptor/reader.py
@@ -51,7 +51,7 @@ def _get_raw_tar_descriptors():
       for tar_entry in tar_file:
         if tar_entry.isfile():
           entry = tar_file.extractfile(tar_entry)
-          raw_descriptors.append(entry.read().strip())
+          raw_descriptors.append(entry.read())
           entry.close()
     
     TAR_DESCRIPTORS = raw_descriptors
@@ -149,7 +149,7 @@ class TestDescriptorReader(unittest.TestCase):
     
     descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "example_descriptor")
     with open(descriptor_path) as descriptor_file:
-      descriptor_entries.append(descriptor_file.read().strip())
+      descriptor_entries.append(descriptor_file.read())
     
     # running this test multiple times to flush out concurrency issues
     for i in xrange(15):



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits