[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [stem/master] Hybrid approach between iterator and attribute
commit f9a2ef8d0dd60a73c7cfe5a329b20329c4b4218c
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date: Sun Aug 19 11:54:29 2012 -0700
Hybrid approach between iterator and attribute
Ok, changed my mind. At least partly. If the user is using parse_file() then we
want to provide them with router entries and a bare document, like I was.
However, if they're calling the document constructors then they've already read
the complete document into memory so there's little reason not to parse it.
In short, documents now have a 'routers' attribute. That attribute is filled
with the parsed routers if we use the constructor directly, and left empty if
we used parse_file().
---
stem/descriptor/networkstatus.py | 75 +++++++++++++++++++++++++-------
test/integ/descriptor/networkstatus.py | 8 ++--
2 files changed, 63 insertions(+), 20 deletions(-)
diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py
index 5054628..a91dcc5 100644
--- a/stem/descriptor/networkstatus.py
+++ b/stem/descriptor/networkstatus.py
@@ -76,7 +76,9 @@ Flag = stem.util.enum.Enum(
def parse_file(document_file, validate = True, is_microdescriptor = False):
"""
- Parses a network status document and provides a NetworkStatusDocument object.
+ Parses a network status and iterates over the RouterDescriptor or
+ RouterMicrodescriptor in it. The document that these instances reference have
+ an empty 'rotuers' attribute to allow for limited memory usage.
:param file document_file: file with network status document content
:param bool validate: checks the validity of the document's contents if True, skips these checks otherwise
@@ -89,29 +91,31 @@ def parse_file(document_file, validate = True, is_microdescriptor = False):
* IOError if the file can't be read
"""
+ header, footer, routers_end = _get_document_content(document_file, validate)
+ document_data = "".join(header + footer)
+
if not is_microdescriptor:
- document_type, router_type = NetworkStatusDocument, RouterDescriptor
+ document = NetworkStatusDocument(document_data, validate)
+ router_type = RouterDescriptor
else:
- document_type, router_type = MicrodescriptorConsensus, RouterMicrodescriptor
-
- document, routers_start, routers_end = _get_document(document_file, validate, document_type)
- document_file.seek(routers_start)
+ document = MicrodescriptorConsensus(document_data, validate)
+ router_type = RouterMicrodescriptor
- while document_file.tell() < routers_end:
- desc_content = "".join(_read_until_keywords("r", document_file, ignore_first = True, end_position = routers_end))
- yield router_type(desc_content, document, validate)
+ for desc in _get_routers(document_file, validate, document, routers_end, router_type):
+ yield desc
-def _get_document(document_file, validate, document_type):
+def _get_document_content(document_file, validate):
"""
Network status documents consist of three sections: header, router entries,
and the footer. This provides back a tuple with the following...
- (NetworkStatusDocument, routers_start, routers_end)
+ (header_lines, footer_lines, routers_end)
+
+ This leaves the document_file at the start of the router entries.
:param file document_file: file with network status document content
:param bool validate: checks the validity of the document's contents if True, skips these checks otherwise
- :param object document_type: consensus document class to construct
- :returns: tuple with the network status document and range that has the routers
+ :returns: tuple with the network status document content and ending position of the routers
:raises:
* ValueError if the contents is malformed and validate is True
@@ -132,9 +136,31 @@ def _get_document(document_file, validate, document_type):
routers_end = document_file.tell()
footer = document_file.readlines()
- document_data = "".join(header + footer)
+ document_file.seek(routers_start)
+ return (header, footer, routers_end)
+
+def _get_routers(document_file, validate, document, end_position, router_type):
+ """
+ Iterates over the router entries in a given document. The document_file is
+ expected to be at the start of the router section and the end_position
+ desigates where that section ends.
+
+ :param file document_file: file with network status document content
+ :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise
+ :param object document: document the descriptors originate from
+ :param int end_position: location in the document_file where the router section ends
+ :param class router_type: router class to construct
- return (document_type(document_data, validate), routers_start, routers_end)
+ :returns: iterator over router_type instances
+
+ :raises:
+ * ValueError if the contents is malformed and validate is True
+ * IOError if the file can't be read
+ """
+
+ while document_file.tell() < end_position:
+ desc_content = "".join(_read_until_keywords("r", document_file, ignore_first = True, end_position = end_position))
+ yield router_type(desc_content, document, validate)
class NetworkStatusDocument(stem.descriptor.Descriptor):
"""
@@ -142,6 +168,8 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):
This could be a v3 consensus or vote document.
+ :var tuple routers: RouterDescriptor contained in the document
+
:var bool validated: **\*** whether the document is validated
:var str network_status_version: **\*** a document format version. For v3 documents this is "3"
:var str vote_status: **\*** status of the vote (is either "vote" or "consensus")
@@ -198,7 +226,19 @@ class NetworkStatusDocument(stem.descriptor.Descriptor):
self.params = {}
self.bandwidth_weights = {}
- self._parse(raw_content)
+ document_file = StringIO(raw_content)
+ header, footer, routers_end = _get_document_content(document_file, validate)
+
+ document_content = "".join(header + footer)
+ self._parse(document_content)
+
+ if document_file.tell() < routers_end:
+ self.routers = tuple(_get_routers(document_file, validate, self, routers_end, self._get_router_type()))
+ else:
+ self.routers = ()
+
+ def _get_router_type(self):
+ return RouterDescriptor
def _validate_network_status_version(self):
return self.network_status_version == "3"
@@ -586,6 +626,9 @@ class MicrodescriptorConsensus(NetworkStatusDocument):
| **~** attribute appears only in consensuses
"""
+ def _get_router_type(self):
+ return RouterMicrodescriptor
+
def _validate_network_status_version(self):
return self.network_status_version == "3 microdesc"
diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py
index f95270b..d16929d 100644
--- a/test/integ/descriptor/networkstatus.py
+++ b/test/integ/descriptor/networkstatus.py
@@ -74,8 +74,8 @@ class TestNetworkStatusDocument(unittest.TestCase):
descriptor_path = test.integ.descriptor.get_resource("cached-consensus")
descriptor_file = file(descriptor_path)
- router1 = next(stem.descriptor.networkstatus.parse_file(descriptor_file))
- desc = router1.document
+ desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read())
+ router1 = desc.routers[0]
descriptor_file.close()
self.assertEquals(True, desc.validated)
@@ -167,8 +167,8 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY=
descriptor_path = test.integ.descriptor.get_resource("vote")
descriptor_file = file(descriptor_path)
- router1 = next(stem.descriptor.networkstatus.parse_file(descriptor_file))
- desc = router1.document
+ desc = stem.descriptor.networkstatus.NetworkStatusDocument(descriptor_file.read())
+ router1 = desc.routers[0]
descriptor_file.close()
self.assertEquals(True, desc.validated)
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits