[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [stem/master] Parse bandwidth file headers



commit ecfdba922026cc56ca451d58e7933b42fab9ecc7
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date:   Sat Jan 19 16:11:36 2019 -0800

    Parse bandwidth file headers
    
    Reading all the header metadata fields. Honestly I'm a bit fuzzy on
    the definition of 'eligible', but that's fine. Taking my best guess
    at describing these.
    
      00:08 <+atagar> juga: Minor thing that is crossing my mind while implementing
            stem's bandwidth-file parser is "how does the 'eligible' count differ
            from the number of relay measurements we list?". In other words, why
            does the header provide a 'consensus, eligible, and mininum eligible'
            count rather than a 'consensus, measured, and minimum measured' count.
            'Eligible' is defined as 'having enough measurements' but not
            necessarily having a metric included, so I'm guessing we chose this
            term because there are relays that have metrics (making it 'eligible')
            but are excluded from the results, and need to count toward the
            'minimum percent of the consensus'.
    
      00:09 <+atagar> Anywho, unimportant to parsing. Just a spot in the spec that
            might benefit from a little clarification.
---
 stem/descriptor/bandwidth_metric.py      | 94 ++++++++++++++++++++++++++------
 test/unit/descriptor/bandwidth_metric.py | 40 ++++++++++++++
 2 files changed, 116 insertions(+), 18 deletions(-)

diff --git a/stem/descriptor/bandwidth_metric.py b/stem/descriptor/bandwidth_metric.py
index bb732f4a..900a9d2e 100644
--- a/stem/descriptor/bandwidth_metric.py
+++ b/stem/descriptor/bandwidth_metric.py
@@ -16,9 +16,52 @@ Parsing for Bandwidth Authority metrics as described in Tor's
 
 import datetime
 
-from stem.descriptor import (
-  Descriptor,
-)
+import stem.util.str_tools
+
+from stem.descriptor import Descriptor
+
+
+# Converters header attributes to a given type. Malformed fields should be
+# ignored according to the spec.
+
+def _str(val):
+  return val  # already a str
+
+
+def _int(val):
+  return int(val) if (val and val.isdigit()) else None
+
+
+def _date(val):
+  try:
+    return stem.util.str_tools._parse_iso_timestamp(val)
+  except ValueError:
+    return None  # not an iso formatted date
+
+
+# mapping of attributes => (header, type)
+
+HEADER_ATTR = {
+  'version': ('version', _str),
+
+  'software': ('software', _str),
+  'software_version': ('software_version', _str),
+
+  'earliest_bandwidth': ('earliest_bandwidth', _date),
+  'latest_bandwidth': ('latest_bandwidth', _date),
+  'created_at': ('file_created', _date),
+  'generated_at': ('generator_started', _date),
+
+  'consensus_size': ('number_consensus_relays', _int),
+  'eligible_count': ('number_eligible_relays', _int),
+  'eligible_percent': ('percent_eligible_relays', _int),
+  'min_count': ('minimum_number_eligible_relays', _int),
+  'min_percent': ('minimum_percent_eligible_relays', _int),
+}
+
+HEADER_DEFAULT = {
+  'version': '1.0.0',  # version field was added in 1.1.0
+}
 
 
 def _parse_file(descriptor_file, validate = False, **kwargs):
@@ -42,8 +85,14 @@ def _parse_file(descriptor_file, validate = False, **kwargs):
 
 def _parse_header(descriptor, entries):
   header = {}
+  lines = str(descriptor).split('\n')
+
+  # skip the first line, which should be the timestamp
+
+  if lines and lines[0].isdigit():
+    lines = lines[1:]
 
-  for line in str(descriptor).split('\n'):
+  for line in lines:
     if line == '=====':
       break
     elif line.startswith('node_id='):
@@ -51,15 +100,15 @@ def _parse_header(descriptor, entries):
 
     if '=' in line:
       key, value = line.split('=', 1)
-    elif line.isdigit() and 'timestamp' not in header:
-      key, value = 'timestamp', line
+      header[key] = value
     else:
       raise ValueError("Header expected to be key=value pairs, but had '%s'" % line)
 
-    header[key] = value
-
   descriptor.header = header
 
+  for attr, (keyword, cls) in HEADER_ATTR.items():
+    setattr(descriptor, attr, cls(header.get(keyword, HEADER_DEFAULT.get(attr))))
+
 
 def _parse_timestamp(descriptor, entries):
   first_line = str(descriptor).split('\n', 1)[0]
@@ -70,33 +119,42 @@ def _parse_timestamp(descriptor, entries):
     raise ValueError("First line should be a unix timestamp, but was '%s'" % first_line)
 
 
-def _header_attr(name):
-  def _parse(descriptor, entries):
-    val = descriptor.header.get(name, None)
-    setattr(descriptor, name, val)
-
-  return _parse
-
-
 class BandwidthMetric(Descriptor):
   """
   Tor bandwidth authroity measurements.
 
   :var datetime timestamp: **\*** time when these metrics were published
+  :var str version: **\*** document format version
+
+  :var str software: application that generated these metrics
+  :var str software_version: version of the application that generated these metrics
 
-  :var dict header: **\*** header metadata attributes
+  :var datetime earliest_bandwidth: time of the first sampling
+  :var datetime latest_bandwidth: time of the last sampling
+  :var datetime created_at: time when this file was created
+  :var datetime generated_at: time when collection of these metrics started
+
+  :var int consensus_size: number of relays in the consensus
+  :var int eligible_count: relays with enough measurements to be included
+  :var int eligible_percent: percentage of consensus with enough measurements
+  :var int min_count: minimum eligible relays for results to be provided
+  :var int min_percent: minimum measured percentage of the consensus
+
+  :var dict header: **\*** header metadata
 
   **\*** attribute is either required when we're parsed with validation or has
   a default value, others are left as **None** if undefined
   """
 
-  TYPE_ANNOTATION_NAME = 'badnwidth-file'  # TODO: needs an official @type
+  TYPE_ANNOTATION_NAME = 'badnwidth-file'  # TODO: needs an official @type, https://trac.torproject.org/projects/tor/ticket/28615
 
   ATTRIBUTES = {
     'timestamp': (None, _parse_timestamp),
     'header': ({}, _parse_header),
   }
 
+  ATTRIBUTES.update(dict([(k, (None, _parse_header)) for k in HEADER_ATTR.keys()]))
+
   def __init__(self, raw_content, validate = False):
     super(BandwidthMetric, self).__init__(raw_content, lazy_load = not validate)
 
diff --git a/test/unit/descriptor/bandwidth_metric.py b/test/unit/descriptor/bandwidth_metric.py
index 489f4d41..45739ed7 100644
--- a/test/unit/descriptor/bandwidth_metric.py
+++ b/test/unit/descriptor/bandwidth_metric.py
@@ -18,4 +18,44 @@ class TestBandwidthMetric(unittest.TestCase):
     """
 
     desc = list(stem.descriptor.parse_file(test.unit.descriptor.get_resource('bwauth_v1.0'), 'badnwidth-file 1.0'))[0]
+
     self.assertEqual(datetime.datetime(2019, 1, 14, 17, 41, 29), desc.timestamp)
+    self.assertEqual('1.0.0', desc.version)
+
+    self.assertEqual(None, desc.software)
+    self.assertEqual(None, desc.software_version)
+
+    self.assertEqual(None, desc.earliest_bandwidth)
+    self.assertEqual(None, desc.latest_bandwidth)
+    self.assertEqual(None, desc.created_at)
+    self.assertEqual(None, desc.generated_at)
+
+    self.assertEqual(None, desc.consensus_size)
+    self.assertEqual(None, desc.eligible_count)
+    self.assertEqual(None, desc.eligible_percent)
+    self.assertEqual(None, desc.min_count)
+    self.assertEqual(None, desc.min_percent)
+
+  def test_format_v1_2(self):
+    """
+    Parse version 1.2 formatted metrics.
+    """
+
+    desc = list(stem.descriptor.parse_file(test.unit.descriptor.get_resource('bwauth_v1.2'), 'badnwidth-file 1.2'))[0]
+
+    self.assertEqual(datetime.datetime(2019, 1, 14, 5, 34, 59), desc.timestamp)
+    self.assertEqual('1.2.0', desc.version)
+
+    self.assertEqual('sbws', desc.software)
+    self.assertEqual('1.0.2', desc.software_version)
+
+    self.assertEqual(datetime.datetime(2019, 1, 4, 5, 35, 29), desc.earliest_bandwidth)
+    self.assertEqual(datetime.datetime(2019, 1, 14, 5, 34, 59), desc.latest_bandwidth)
+    self.assertEqual(datetime.datetime(2019, 1, 14, 5, 35, 6), desc.created_at)
+    self.assertEqual(datetime.datetime(2019, 1, 3, 22, 45, 8), desc.generated_at)
+
+    self.assertEqual(6514, desc.consensus_size)
+    self.assertEqual(6256, desc.eligible_count)
+    self.assertEqual(96, desc.eligible_percent)
+    self.assertEqual(3908, desc.min_count)
+    self.assertEqual(60, desc.min_percent)



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits