[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [stem/master] Use new CollecTor index fields



commit 1543c078acd2b3270e8d72b62fad28fe357f5481
Author: Damian Johnson <atagar@xxxxxxxxxxxxxx>
Date:   Sat Nov 30 19:14:53 2019 -0800

    Use new CollecTor index fields
    
    Thanks to Karsten CollecTor now provides descriptor types, checksums, and time
    ranges...
    
      https://trac.torproject.org/projects/tor/ticket/31204
    
    As such we no longer need to guess these based on the filename (hazaa!).
---
 stem/descriptor/collector.py                   | 125 ++--
 test/unit/descriptor/collector.py              | 137 +++-
 test/unit/descriptor/data/collector/index.json |   2 +-
 test/unit/descriptor/data/collector/index.py   | 910 ++++++++++++++++++-------
 4 files changed, 826 insertions(+), 348 deletions(-)

diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py
index 28fcbd49..05826c49 100644
--- a/stem/descriptor/collector.py
+++ b/stem/descriptor/collector.py
@@ -48,7 +48,10 @@ With CollecTor you can either read descriptors directly...
 .. versionadded:: 1.8.0
 """
 
+import base64
+import binascii
 import datetime
+import hashlib
 import json
 import os
 import re
@@ -73,40 +76,6 @@ SEC_DATE = re.compile('(\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2})')
 
 FUTURE = datetime.datetime(9999, 1, 1)
 
-# mapping of path prefixes to their descriptor type (sampled 7/11/19)
-
-COLLECTOR_DESC_TYPES = {
-  'archive/bridge-descriptors/server-descriptors/': 'bridge-server-descriptor 1.2',
-  'archive/bridge-descriptors/extra-infos/': 'bridge-extra-info 1.3',
-  'archive/bridge-descriptors/statuses/': 'bridge-network-status 1.1',
-  'archive/bridge-pool-assignments/': 'bridge-pool-assignment 1.0',
-  'archive/exit-lists/': 'tordnsel 1.0',
-  'archive/relay-descriptors/bandwidths/': 'bandwidth-file 1.0',
-  'archive/relay-descriptors/certs': 'dir-key-certificate-3 1.0',
-  'archive/relay-descriptors/consensuses/': 'network-status-consensus-3 1.0',
-  'archive/relay-descriptors/extra-infos/': 'extra-info 1.0',
-  'archive/relay-descriptors/microdescs/': ('network-status-microdesc-consensus-3 1.0', 'microdescriptor 1.0'),
-  'archive/relay-descriptors/server-descriptors/': 'server-descriptor 1.0',
-  'archive/relay-descriptors/statuses/': 'network-status-2 1.0',
-  'archive/relay-descriptors/tor/': 'directory 1.0',
-  'archive/relay-descriptors/votes/': 'network-status-vote-3 1.0',
-  'archive/torperf/': 'torperf 1.0',
-  'archive/webstats/': (),
-  'recent/bridge-descriptors/extra-infos/': 'bridge-extra-info 1.3',
-  'recent/bridge-descriptors/server-descriptors/': 'bridge-server-descriptor 1.2',
-  'recent/bridge-descriptors/statuses/': 'bridge-network-status 1.2',
-  'recent/exit-lists/': 'tordnsel 1.0',
-  'recent/relay-descriptors/bandwidths/': 'bandwidth-file 1.0',
-  'recent/relay-descriptors/consensuses/': 'network-status-consensus-3 1.0',
-  'recent/relay-descriptors/extra-infos/': 'extra-info 1.0',
-  'recent/relay-descriptors/microdescs/consensus-microdesc/': 'network-status-microdesc-consensus-3 1.0',
-  'recent/relay-descriptors/microdescs/micro/': 'microdescriptor 1.0',
-  'recent/relay-descriptors/server-descriptors/': 'server-descriptor 1.0',
-  'recent/relay-descriptors/votes/': 'network-status-vote-3 1.0',
-  'recent/torperf/': 'torperf 1.1',
-  'recent/webstats/': (),
-}
-
 
 def get_instance():
   """
@@ -206,9 +175,11 @@ class File(object):
   File within CollecTor.
 
   :var str path: file path within collector
+  :var tuple types: descriptor types contained within this file
   :var stem.descriptor.Compression compression: file compression, **None** if
     this cannot be determined
   :var int size: size of the file
+  :var str sha256: file's sha256 checksum
 
   :var datetime start: beginning of the time range descriptors are for,
     **None** if this cannot be determined
@@ -217,17 +188,24 @@ class File(object):
   :var datetime last_modified: when the file was last modified
   """
 
-  def __init__(self, path, size, last_modified):
+  def __init__(self, path, types, size, sha256, first_published, last_published, last_modified):
     self.path = path
+    self.types = tuple(types) if types else ()
     self.compression = File._guess_compression(path)
     self.size = size
-
-    self.start, self.end = File._guess_time_range(path)
+    self.sha256 = sha256
     self.last_modified = datetime.datetime.strptime(last_modified, '%Y-%m-%d %H:%M')
-
-    self._guessed_type = File._guess_descriptor_types(path)
     self._downloaded_to = None  # location we last downloaded to
 
+    # Most descriptor types have publication time fields, but microdescriptors
+    # don't because these files lack timestamps to parse.
+
+    if first_published and last_published:
+      self.start = datetime.datetime.strptime(first_published, '%Y-%m-%d %H:%M')
+      self.end = datetime.datetime.strptime(last_published, '%Y-%m-%d %H:%M')
+    else:
+      self.start, self.end = File._guess_time_range(path)
+
   def read(self, directory = None, descriptor_type = None, document_handler = DocumentHandler.ENTRIES, timeout = None, retries = 3):
     """
     Provides descriptors from this archive. Descriptors are downloaded or read
@@ -267,12 +245,18 @@ class File(object):
     """
 
     if descriptor_type is None:
-      if not self._guessed_type:
-        raise ValueError("Unable to determine this file's descriptor type")
-      elif len(self._guessed_type) > 1:
-        raise ValueError("Unable to determine disambiguate file's descriptor type from %s" % ', '.join(self._guessed_type))
+      # If archive contains multiple descriptor types the caller must provide a
+      # 'descriptor_type' argument so we can disambiguate. However, if only the
+      # version number varies we can probably simply pick one.
 
-      descriptor_type = self._guessed_type[0]
+      base_types = set([t.split(' ')[0] for t in self.types])
+
+      if not self.types:
+        raise ValueError("Unable to determine this file's descriptor type")
+      elif len(base_types) > 1:
+        raise ValueError("Unable to disambiguate file's descriptor type from among %s" % ', '.join(self.types))
+      else:
+        descriptor_type = self.types[0]
 
     if directory is None:
       if self._downloaded_to and os.path.exists(self._downloaded_to):
@@ -299,7 +283,7 @@ class File(object):
       if descriptor_type is None or descriptor_type.startswith(desc.type_annotation().name):
         yield desc
 
-  def download(self, directory, decompress = True, timeout = None, retries = 3):
+  def download(self, directory, decompress = True, timeout = None, retries = 3, overwrite = False):
     """
     Downloads this file to the given location. If a file already exists this is
     a no-op.
@@ -309,17 +293,16 @@ class File(object):
     :param int timeout: timeout when connection becomes idle, no timeout
       applied if **None**
     :param int retries: maximum attempts to impose
+    :param bool overwrite: if this file exists but mismatches CollecTor's
+      checksum then overwrites if **True**, otherwise rases an exception
 
     :returns: **str** with the path we downloaded to
 
-    :raises: :class:`~stem.DownloadFailed` if the download fails
+    :raises:
+      * :class:`~stem.DownloadFailed` if the download fails
+      * **IOError** if a mismatching file exists and **overwrite** is **False**
     """
 
-    # TODO: If checksums get added to the index we should replace
-    # the path check below to verify that...
-    #
-    #   https://trac.torproject.org/projects/tor/ticket/31204
-
     filename = self.path.split('/')[-1]
 
     if self.compression != Compression.PLAINTEXT and decompress:
@@ -331,8 +314,18 @@ class File(object):
 
     if not os.path.exists(directory):
       os.makedirs(directory)
-    elif os.path.exists(path):
-      return path  # file already exists
+
+    # check if this file already exists with the correct checksum
+
+    if os.path.exists(path):
+      with open(path) as prior_file:
+        expected_hash = binascii.hexlify(base64.b64decode(self.sha256))
+        actual_hash = hashlib.sha256(prior_file.read()).hexdigest()
+
+        if expected_hash == actual_hash:
+          return path  # nothing to do, we already have the file
+        elif not overwrite:
+          raise IOError("%s already exists but mismatches CollecTor's checksum (expected: %s, actual: %s)" % (path, expected_hash, actual_hash))
 
     response = stem.util.connection.download(COLLECTOR_URL + self.path, timeout, retries)
 
@@ -346,25 +339,6 @@ class File(object):
     return path
 
   @staticmethod
-  def _guess_descriptor_types(path):
-    """
-    Descriptor @type this file is expected to have based on its path. If unable
-    to determine any this tuple is empty.
-
-    Hopefully this will be replaced with an explicit value in the future:
-
-      https://trac.torproject.org/projects/tor/ticket/31204
-
-    :returns: **tuple** with the descriptor types this file is expected to have
-    """
-
-    for path_prefix, types in COLLECTOR_DESC_TYPES.items():
-      if path.startswith(path_prefix):
-        return (types,) if isinstance(types, str) else types
-
-    return ()
-
-  @staticmethod
   def _guess_compression(path):
     """
     Determine file comprssion from CollecTor's filename.
@@ -636,8 +610,7 @@ class CollecTor(object):
     :param descriptor.Compression compression: compression type to
       download from, if undefiled we'll use the best decompression available
 
-    :returns: :class:`~stem.descriptor.collector.Index` with the archive
-      contents
+    :returns: **dict** with the archive contents
 
     :raises:
       If unable to retrieve the index this provide...
@@ -694,7 +667,7 @@ class CollecTor(object):
       elif end and (f.end is None or f.end > end):
         continue
 
-      if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in f._guessed_type]):
+      if descriptor_type is None or any([desc_type.startswith(descriptor_type) for desc_type in f.types]):
         matches.append(f)
 
     return matches
@@ -719,7 +692,7 @@ class CollecTor(object):
       if k == 'files':
         for attr in v:
           file_path = '/'.join(path + [attr.get('path')])
-          files.append(File(file_path, attr.get('size'), attr.get('last_modified')))
+          files.append(File(file_path, attr.get('types'), attr.get('size'), attr.get('sha256'), attr.get('first_published'), attr.get('last_published'), attr.get('last_modified')))
       elif k == 'directories':
         for attr in v:
           files.extend(CollecTor._files(attr, path + [attr.get('path')]))
diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py
index 44893fab..7d80d572 100644
--- a/test/unit/descriptor/collector.py
+++ b/test/unit/descriptor/collector.py
@@ -29,17 +29,6 @@ with open(get_resource('collector/index.json'), 'rb') as index_file:
 class TestCollector(unittest.TestCase):
   # tests for the File class
 
-  def test_file_guess_descriptor_types(self):
-    test_values = {
-      'archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar.xz': ('bridge-extra-info 1.3',),
-      'archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz': ('network-status-microdesc-consensus-3 1.0', 'microdescriptor 1.0'),
-      'archive/webstats/webstats-2015-03.tar': (),
-      'archive/no_such_file.tar': (),
-    }
-
-    for path, expected in test_values.items():
-      self.assertEqual(expected, File._guess_descriptor_types(path))
-
   def test_file_guess_compression(self):
     test_values = {
       'archive/relay-descriptors/microdescs/microdescs-2014-01.tar.xz': Compression.LZMA,
@@ -65,7 +54,7 @@ class TestCollector(unittest.TestCase):
     }
 
     for path, (expected_start, expected_end) in test_values.items():
-      f = File(path, 7515396, '2014-02-07 03:59')
+      f = File(path, [], 7515396, 'BVVDEkegsLzkAn30dYikr4yTT79+XScfc0VUVEb83tM=', None, None, '2019-07-29 18:45')
       self.assertEqual(expected_start, f.start)
       self.assertEqual(expected_end, f.end)
 
@@ -154,7 +143,7 @@ class TestCollector(unittest.TestCase):
   def test_files(self):
     collector = CollecTor()
     files = collector.files()
-    self.assertEqual(85, len(files))
+    self.assertEqual(96, len(files))
 
     extrainfo_file = list(filter(lambda x: x.path.endswith('extra-infos-2007-09.tar.xz'), files))[0]
     self.assertEqual('archive/relay-descriptors/extra-infos/extra-infos-2007-09.tar.xz', extrainfo_file.path)
@@ -170,9 +159,9 @@ class TestCollector(unittest.TestCase):
       'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz',
       'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz',
       'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz',
-      'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors',
-      'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors',
-      'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors',
+      'recent/relay-descriptors/server-descriptors/2019-11-28-01-05-00-server-descriptors',
+      'recent/relay-descriptors/server-descriptors/2019-11-28-00-05-00-server-descriptors',
+      'recent/relay-descriptors/server-descriptors/2019-11-27-23-05-00-server-descriptors',
     ], [f.path for f in collector.files(descriptor_type = 'server-descriptor')])
 
   @patch('stem.descriptor.collector.CollecTor.index', Mock(return_value = EXAMPLE_INDEX))
@@ -180,9 +169,9 @@ class TestCollector(unittest.TestCase):
     collector = CollecTor()
 
     self.assertEqual([
-      'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors',
-      'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors',
-      'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors',
+      'recent/relay-descriptors/server-descriptors/2019-11-28-01-05-00-server-descriptors',
+      'recent/relay-descriptors/server-descriptors/2019-11-28-00-05-00-server-descriptors',
+      'recent/relay-descriptors/server-descriptors/2019-11-27-23-05-00-server-descriptors',
     ], [f.path for f in collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2007, 1, 1))])
 
     self.assertEqual([
@@ -201,7 +190,15 @@ class TestCollector(unittest.TestCase):
     with open(get_resource('collector/server-descriptors-2005-12-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
 
-    files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar', 12345, '2016-09-04 09:21')]
+    files_mock.return_value = [stem.descriptor.collector.File(
+      'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar',
+      ['server-descriptor 1.0'],
+      1348620,
+      'v3ANi2FD4xAhmyzigQq9gvlLwpXH8I6fGoiYlWLjOy8=',
+      '2005-12-15 01:42',
+      '2005-12-17 11:06',
+      '2016-06-24 08:12',
+    )]
 
     descriptors = list(stem.descriptor.collector.get_server_descriptors())
     self.assertEqual(5, len(descriptors))
@@ -216,7 +213,15 @@ class TestCollector(unittest.TestCase):
     with open(get_resource('collector/bridge-server-descriptors-2019-02-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
 
-    files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/server-descriptors/bridge-server-descriptors-2019-02.tar', 12345, '2016-09-04 09:21')]
+    files_mock.return_value = [stem.descriptor.collector.File(
+      'archive/bridge-descriptors/server-descriptors/bridge-server-descriptors-2008-05.tar',
+      ['bridge-server-descriptor 1.2'],
+      205348,
+      'NRb2dzS2OhFKYjfr5WoleOFokqC4C+qf0Nu4iegnFLo=',
+      '2008-05-14 18:22',
+      '2008-05-31 23:09',
+      '2016-09-09 14:13',
+    )]
 
     descriptors = list(stem.descriptor.collector.get_server_descriptors(bridge = True))
     self.assertEqual(4, len(descriptors))
@@ -231,7 +236,15 @@ class TestCollector(unittest.TestCase):
     with open(get_resource('collector/extra-infos-2019-04-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
 
-    files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/extra-infos/extra-infos-2019-04.tar', 12345, '2016-09-04 09:21')]
+    files_mock.return_value = [stem.descriptor.collector.File(
+      'archive/relay-descriptors/extra-infos/extra-infos-2007-08.tar',
+      ['extra-info 1.0'],
+      3016916,
+      'UcAIrzYjFU52mRHXNle/fbI21lvfsVkeC0NpBZ/Pt/w=',
+      '2007-08-14 17:35',
+      '2007-08-31 23:53',
+      '2016-06-23 09:53',
+    )]
 
     descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors())
     self.assertEqual(7, len(descriptors))
@@ -246,7 +259,15 @@ class TestCollector(unittest.TestCase):
     with open(get_resource('collector/bridge-extra-infos-2019-03-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
 
-    files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/extra-infos/bridge-extra-infos-2019-03.tar', 12345, '2016-09-04 09:21')]
+    files_mock.return_value = [stem.descriptor.collector.File(
+      'archive/bridge-descriptors/extra-infos/bridge-extra-infos-2008-05.tar',
+      ['bridge-extra-info 1.3'],
+      377644,
+      'aDD2q7uNGOM+WuH67+nTd7rvFN4P580xPAmXYtqxr2I=',
+      '2008-05-13 15:21',
+      '2008-05-31 23:09',
+      '2016-09-04 09:21',
+    )]
 
     descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors(bridge = True))
     self.assertEqual(6, len(descriptors))
@@ -261,7 +282,15 @@ class TestCollector(unittest.TestCase):
     with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
 
-    files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/microdescs/microdescs-2019-05.tar', 12345, '2016-09-04 09:21')]
+    files_mock.return_value = [stem.descriptor.collector.File(
+      'archive/relay-descriptors/microdescs/microdescs-2014-01.tar',
+      ['microdescriptor 1.0', 'network-status-microdesc-consensus-3 1.0'],
+      7515396,
+      'DFugbV1phhpiEB0QeyyueKp0V/bicmAAkdBk/95RjKk=',
+      '2014-01-22 09:00',
+      '2014-01-31 23:00',
+      '2014-02-07 03:59',
+    )]
 
     descriptors = list(stem.descriptor.collector.get_microdescriptors())
     self.assertEqual(3, len(descriptors))
@@ -276,7 +305,15 @@ class TestCollector(unittest.TestCase):
     with open(get_resource('collector/consensuses-2018-06-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
 
-    files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/consensuses/consensuses-2018-06.tar', 12345, '2016-09-04 09:21')]
+    files_mock.return_value = [stem.descriptor.collector.File(
+      'archive/relay-descriptors/consensuses/2019-11-27-23-00-00-consensus.tar',
+      ['network-status-consensus-3 1.0'],
+      2208505,
+      'cGWT19Y0UVE/EUi3ZayacGvJU5t9T6MKaTOrNarAqlI=',
+      '2019-11-27 23:00',
+      '2019-11-27 23:00',
+      '2019-11-27 23:05',
+    )]
 
     descriptors = list(stem.descriptor.collector.get_consensus())
     self.assertEqual(243, len(descriptors))
@@ -303,7 +340,15 @@ class TestCollector(unittest.TestCase):
     with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
 
-    files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/microdescs/microdescs-2019-05.tar', 12345, '2016-09-04 09:21')]
+    files_mock.return_value = [stem.descriptor.collector.File(
+      'archive/relay-descriptors/microdescs/microdescs-2014-01.tar',
+      ['microdescriptor 1.0', 'network-status-microdesc-consensus-3 1.0'],
+      7515396,
+      'DFugbV1phhpiEB0QeyyueKp0V/bicmAAkdBk/95RjKk=',
+      '2014-01-22 09:00',
+      '2014-01-31 23:00',
+      '2014-02-07 03:59',
+    )]
 
     descriptors = list(stem.descriptor.collector.get_consensus(microdescriptor = True))
     self.assertEqual(556, len(descriptors))
@@ -318,7 +363,15 @@ class TestCollector(unittest.TestCase):
     with open(get_resource('collector/bridge-statuses-2019-05-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
 
-    files_mock.return_value = [stem.descriptor.collector.File('archive/bridge-descriptors/microdescs/bridge-statuses-2019-05.tar', 12345, '2016-09-04 09:21')]
+    files_mock.return_value = [stem.descriptor.collector.File(
+      'archive/bridge-descriptors/microdescs/bridge-statuses-2008-05.tar',
+      ['bridge-network-status 1.1'],
+      74792,
+      'scynC2b8xKD+NbkejGK7mKCegUwGPwgzXu7MouxBSj0=',
+      '2008-05-16 19:46',
+      '2008-05-31 23:37',
+      '2016-09-14 21:11',
+    )]
 
     descriptors = list(stem.descriptor.collector.get_consensus(bridge = True))
     self.assertEqual(2593, len(descriptors))
@@ -333,7 +386,15 @@ class TestCollector(unittest.TestCase):
     with open(get_resource('collector/certs-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
 
-    files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/certs.tar', 12345, '2016-09-04 09:21')]
+    files_mock.return_value = [stem.descriptor.collector.File(
+      'archive/relay-descriptors/certs.tar',
+      ['dir-key-certificate-3 1.0'],
+      151748,
+      'ZfcE9RJwHvXhXaZ2xDzpoOJFqJeQR5ovOePOyNkKDi8=',
+      '2007-09-19 03:14',
+      '2019-10-08 04:06',
+      '2019-11-29 03:33',
+    )]
 
     descriptors = list(stem.descriptor.collector.get_key_certificates())
     self.assertEqual(5, len(descriptors))
@@ -348,7 +409,15 @@ class TestCollector(unittest.TestCase):
     with open(get_resource('collector/bandwidths-2019-05-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
 
-    files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/bandwidths/bandwidths-2019-05.tar', 12345, '2016-09-04 09:21')]
+    files_mock.return_value = [stem.descriptor.collector.File(
+      'archive/relay-descriptors/bandwidths/bandwidths-2017-08.tar',
+      ['bandwidth-file 1.0'],
+      13330020,
+      'BVVDEkegsLzkAn30dYikr4yTT79+XScfc0VUVEb83tM=',
+      '2017-08-09 09:35',
+      '2017-08-31 23:35',
+      '2019-07-29 18:45',
+    )]
 
     descriptors = list(stem.descriptor.collector.get_bandwidth_files())
     self.assertEqual(2, len(descriptors))
@@ -363,7 +432,15 @@ class TestCollector(unittest.TestCase):
     with open(get_resource('collector/exit-list-2018-11-cropped.tar'), 'rb') as archive:
       download_mock.return_value = archive.read()
 
-    files_mock.return_value = [stem.descriptor.collector.File('archive/exit-lists/exit-list-2018-11.tar', 12345, '2016-09-04 09:21')]
+    files_mock.return_value = [stem.descriptor.collector.File(
+      'archive/exit-lists/exit-list-2010-02.tar',
+      ['tordnsel 1.0'],
+      272008,
+      'Q6ZAAy7RVbO+8rHH48AEZUU9PqcY5jD9zMASqjMzyns=',
+      '2010-02-22 15:32',
+      '2010-02-28 23:18',
+      '2012-05-31 18:57',
+    )]
 
     descriptors = list(stem.descriptor.collector.get_exit_lists())
     self.assertEqual(3713, len(descriptors))
diff --git a/test/unit/descriptor/data/collector/index.json b/test/unit/descriptor/data/collector/index.json
index c6528f23..d50635d1 100644
--- a/test/unit/descriptor/data/collector/index.json
+++ b/test/unit/descriptor/data/collector/index.json
@@ -1 +1 @@
-{"index_created":"2019-07-06 01:54","build_revision":"df98ac79","path":"https://collector.torproject.org","directories":[{"path":"archive","directories":[{"path":"bridge-descriptors","directories":[{"path":"extra-infos","files":[{"path":"bridge-extra-infos-2008-05.tar.xz","size":377644,"last_modified":"2016-09-04 09:21"},{"path":"bridge-extra-infos-2008-06.tar.xz","size":600484,"last_modified":"2016-09-04 09:21"},{"path":"bridge-extra-infos-2008-07.tar.xz","size":716320,"last_modified":"2016-09-04 09:21"}]},{"path":"server-descriptors","files":[{"path":"bridge-server-descriptors-2008-05.tar.xz","size":205348,"last_modified":"2016-09-09 14:13"},{"path":"bridge-server-descriptors-2008-06.tar.xz","size":342828,"last_modified":"2016-09-09 14:13"},{"path":"bridge-server-descriptors-2008-07.tar.xz","size":374848,"last_modified":"2016-09-09 14:13"}]},{"path":"statuses","files":[{"path":"bridge-statuses-2008-05.tar.xz","size":74792,"last_modified":"2016-09-14 21:11"},{"path":"bridge-statuse
 s-2008-06.tar.xz","size":123488,"last_modified":"2016-09-14 21:11"},{"path":"bridge-statuses-2008-07.tar.xz","size":143836,"last_modified":"2016-09-14 21:11"}]}]},{"path":"bridge-pool-assignments","files":[{"path":"bridge-pool-assignments-2010-09.tar.xz","size":32804,"last_modified":"2012-05-31 10:21"},{"path":"bridge-pool-assignments-2010-10.tar.xz","size":304684,"last_modified":"2012-05-31 10:21"},{"path":"bridge-pool-assignments-2010-11.tar.xz","size":292228,"last_modified":"2012-05-31 10:21"}]},{"path":"exit-lists","files":[{"path":"exit-list-2010-02.tar.xz","size":272008,"last_modified":"2012-05-31 18:57"},{"path":"exit-list-2010-03.tar.xz","size":1247484,"last_modified":"2012-05-31 18:57"},{"path":"exit-list-2010-04.tar.xz","size":1139896,"last_modified":"2012-05-31 18:57"}]},{"path":"relay-descriptors","files":[{"path":"certs.tar.xz","size":144696,"last_modified":"2019-07-03 03:29"}],"directories":[{"path":"bandwidths","files":[{"path":"bandwidths-2019-05.tar.xz","size":50385
 816,"last_modified":"2019-06-07 08:05"},{"path":"bandwidths-2019-06.tar.xz","size":105881156,"last_modified":"2019-07-03 07:30"},{"path":"bandwidths-2019-07.tar.xz","size":11374436,"last_modified":"2019-07-03 07:12"}]},{"path":"consensuses","files":[{"path":"consensuses-2007-10.tar.xz","size":1061648,"last_modified":"2012-05-15 14:35"},{"path":"consensuses-2007-11.tar.xz","size":6810308,"last_modified":"2012-05-15 14:35"},{"path":"consensuses-2007-12.tar.xz","size":8106968,"last_modified":"2012-05-15 14:35"}]},{"path":"extra-infos","files":[{"path":"extra-infos-2007-08.tar.xz","size":3016916,"last_modified":"2016-06-23 09:53"},{"path":"extra-infos-2007-09.tar.xz","size":6459884,"last_modified":"2016-06-23 09:54"},{"path":"extra-infos-2007-10.tar.xz","size":7326564,"last_modified":"2016-06-23 09:54"}]},{"path":"microdescs","files":[{"path":"microdescs-2014-01.tar.xz","size":7515396,"last_modified":"2014-02-07 03:59"},{"path":"microdescs-2014-02.tar.xz","size":21822944,"last_modified"
 :"2014-03-07 04:54"},{"path":"microdescs-2014-03.tar.xz","size":24201436,"last_modified":"2014-04-07 03:54"}]},{"path":"server-descriptors","files":[{"path":"server-descriptors-2005-12.tar.xz","size":1348620,"last_modified":"2016-06-24 08:12"},{"path":"server-descriptors-2006-02.tar.xz","size":28625408,"last_modified":"2016-06-24 08:14"},{"path":"server-descriptors-2006-03.tar.xz","size":49548736,"last_modified":"2016-06-24 08:17"}]},{"path":"statuses","files":[{"path":"statuses-2005-12.tar.xz","size":1468844,"last_modified":"2016-06-25 11:50"},{"path":"statuses-2006-01.tar.xz","size":3344280,"last_modified":"2016-06-25 11:52"},{"path":"statuses-2006-02.tar.xz","size":4006336,"last_modified":"2016-06-25 11:54"}]},{"path":"tor","files":[{"path":"tor-2004-05.tar.xz","size":386672,"last_modified":"2012-05-18 14:26"},{"path":"tor-2004-06.tar.xz","size":1087980,"last_modified":"2012-05-18 14:26"},{"path":"tor-2004-07.tar.xz","size":1366568,"last_modified":"2012-05-18 14:26"}]},{"path":"v
 otes","files":[{"path":"votes-2007-10.tar.xz","size":1356504,"last_modified":"2012-05-15 14:51"},{"path":"votes-2007-11.tar.xz","size":10641492,"last_modified":"2012-05-15 14:51"},{"path":"votes-2007-12.tar.xz","size":14712136,"last_modified":"2012-05-15 14:52"}]}]},{"path":"torperf","files":[{"path":"torperf-2009-07.tar.xz","size":182712,"last_modified":"2012-05-30 07:23"},{"path":"torperf-2009-08.tar.xz","size":203236,"last_modified":"2012-05-30 07:23"},{"path":"torperf-2009-09.tar.xz","size":193832,"last_modified":"2012-05-30 07:23"}]},{"path":"webstats","files":[{"path":"webstats-2015-01.tar","size":30720,"last_modified":"2018-03-19 16:07"},{"path":"webstats-2015-02.tar","size":20480,"last_modified":"2018-03-19 16:07"},{"path":"webstats-2015-03.tar","size":20480,"last_modified":"2018-03-19 16:07"}]}]},{"path":"contrib"},{"path":"recent","directories":[{"path":"bridge-descriptors","directories":[{"path":"extra-infos","files":[{"path":"2019-07-03-02-09-00-extra-infos","size":50781
 6,"last_modified":"2019-07-03 02:09"},{"path":"2019-07-03-03-09-00-extra-infos","size":558790,"last_modified":"2019-07-03 03:09"},{"path":"2019-07-03-04-09-05-extra-infos","size":592279,"last_modified":"2019-07-03 04:09"}]},{"path":"server-descriptors","files":[{"path":"2019-07-03-02-09-00-server-descriptors","size":274355,"last_modified":"2019-07-03 02:09"},{"path":"2019-07-03-03-09-00-server-descriptors","size":295671,"last_modified":"2019-07-03 03:09"},{"path":"2019-07-03-04-09-05-server-descriptors","size":312822,"last_modified":"2019-07-03 04:09"}]},{"path":"statuses","files":[{"path":"20190703-011231-BA44A889E64B93FAA2B114E02C2A279A8555C533","size":238340,"last_modified":"2019-07-03 02:09"},{"path":"20190703-014231-BA44A889E64B93FAA2B114E02C2A279A8555C533","size":238147,"last_modified":"2019-07-03 02:09"},{"path":"20190703-021231-BA44A889E64B93FAA2B114E02C2A279A8555C533","size":238231,"last_modified":"2019-07-03 03:09"}]}]},{"path":"exit-lists","files":[{"path":"2019-07-03-02-
 02-00","size":158516,"last_modified":"2019-07-03 02:02"},{"path":"2019-07-03-03-02-00","size":158830,"last_modified":"2019-07-03 03:02"},{"path":"2019-07-03-04-02-00","size":158831,"last_modified":"2019-07-03 04:02"}]},{"path":"relay-descriptors","directories":[{"path":"bandwidths","files":[{"path":"2019-07-03-00-30-57-bandwidth-616DF2278EE2C90F475E4EA2562E2C00EB9F10E517FB901229F331AEB70B8AD7","size":2580044,"last_modified":"2019-07-03 02:35"},{"path":"2019-07-03-01-35-02-bandwidth-2A5B679E9AA2D5C903CD16C00E16FEFA3E21E38AB3DE338941AE4AC13FD505DE","size":4210146,"last_modified":"2019-07-03 02:35"},{"path":"2019-07-03-01-35-03-bandwidth-3B06DAD9EDE8E1D08CC494E856AFD9D49256ECC2D68456799DD1EC7ED3436875","size":4249699,"last_modified":"2019-07-03 02:35"}]},{"path":"consensuses","files":[{"path":"2019-07-03-02-00-00-consensus","size":2211265,"last_modified":"2019-07-03 02:05"},{"path":"2019-07-03-03-00-00-consensus","size":2204478,"last_modified":"2019-07-03 03:05"},{"path":"2019-07-03-04
 -00-00-consensus","size":2202554,"last_modified":"2019-07-03 04:05"}]},{"path":"extra-infos","files":[{"path":"2019-07-03-02-05-00-extra-infos","size":1162899,"last_modified":"2019-07-03 02:05"},{"path":"2019-07-03-03-05-00-extra-infos","size":1133425,"last_modified":"2019-07-03 03:05"},{"path":"2019-07-03-04-05-00-extra-infos","size":1153793,"last_modified":"2019-07-03 04:05"}]},{"path":"microdescs","directories":[{"path":"consensus-microdesc","files":[{"path":"2019-07-03-02-00-00-consensus-microdesc","size":2028994,"last_modified":"2019-07-03 02:05"},{"path":"2019-07-03-03-00-00-consensus-microdesc","size":2022808,"last_modified":"2019-07-03 03:05"},{"path":"2019-07-03-04-00-00-consensus-microdesc","size":2020751,"last_modified":"2019-07-03 04:05"}]},{"path":"micro","files":[{"path":"2019-07-02-23-05-00-micro","size":19934,"last_modified":"2019-07-02 23:05"},{"path":"2019-07-03-00-05-00-micro","size":12030,"last_modified":"2019-07-03 00:05"},{"path":"2019-07-03-01-05-00-micro","si
 ze":14850,"last_modified":"2019-07-03 01:05"}]}]},{"path":"server-descriptors","files":[{"path":"2019-07-03-02-05-00-server-descriptors","size":1374587,"last_modified":"2019-07-03 02:05"},{"path":"2019-07-03-03-05-00-server-descriptors","size":1353286,"last_modified":"2019-07-03 03:05"},{"path":"2019-07-03-04-05-00-server-descriptors","size":1336125,"last_modified":"2019-07-03 04:05"}]},{"path":"votes","files":[{"path":"2019-07-03-02-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-D9B6923A3AF9FB5DB3CD30191BCBD29A7A936D86","size":3449707,"last_modified":"2019-07-03 02:05"},{"path":"2019-07-03-02-00-00-vote-14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4-39429F0789A9EC07A2A8754C4C449CCABAB787CC","size":3435482,"last_modified":"2019-07-03 02:05"},{"path":"2019-07-03-02-00-00-vote-23D15D965BC35114467363C165C4F724B64B4F66-4337842B697467123B5351E1E8EB64ED497C0EDA","size":3571106,"last_modified":"2019-07-03 02:05"}]}]},{"path":"torperf","files":[{"path":"op-ab-1048576-2019-07-03.tpf","size":3
 9152,"last_modified":"2019-07-04 00:01"},{"path":"op-ab-51200-2019-07-03.tpf","size":248173,"last_modified":"2019-07-04 00:01"},{"path":"op-ab-5242880-2019-07-03.tpf","size":15900,"last_modified":"2019-07-04 00:01"}]},{"path":"webstats","files":[{"path":"2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190620.xz","size":8872,"last_modified":"2019-07-03 04:21"},{"path":"2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190621.xz","size":9312,"last_modified":"2019-07-04 04:21"},{"path":"2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190622.xz","size":9836,"last_modified":"2019-07-05 04:21"}]}]}]}
+{"index_created":"2019-11-30 22:26","build_revision":"199d1bed","path":"https://collector.torproject.org","directories":[{"path":"archive","directories":[{"path":"bridge-descriptors","directories":[{"path":"extra-infos","files":[{"path":"bridge-extra-infos-2008-05.tar.xz","size":377644,"last_modified":"2016-09-04 09:21","types":["bridge-extra-info 1.3"],"first_published":"2008-05-13 15:21","last_published":"2008-05-31 23:09","sha256":"aDD2q7uNGOM+WuH67+nTd7rvFN4P580xPAmXYtqxr2I="},{"path":"bridge-extra-infos-2008-06.tar.xz","size":600484,"last_modified":"2016-09-04 09:21","types":["bridge-extra-info 1.3"],"first_published":"2008-06-01 00:23","last_published":"2008-06-30 23:29","sha256":"VLdUu98Fz8RKTFL13ftA3WJ5lsDJQBRrh/KGLOYAUqk="},{"path":"bridge-extra-infos-2008-07.tar.xz","size":716320,"last_modified":"2016-09-04 09:21","types":["bridge-extra-info 1.3"],"first_published":"2008-07-01 05:06","last_published":"2008-07-31 23:59","sha256":"NNj6mBr/aWDdNZmmPYSERn5pjQh0g04Xu8gUi6kMVds=
 "}]},{"path":"server-descriptors","files":[{"path":"bridge-server-descriptors-2008-05.tar.xz","size":205348,"last_modified":"2016-09-09 14:13","types":["bridge-server-descriptor 1.2"],"first_published":"2008-05-14 18:22","last_published":"2008-05-31 23:09","sha256":"NRb2dzS2OhFKYjfr5WoleOFokqC4C+qf0Nu4iegnFLo="},{"path":"bridge-server-descriptors-2008-06.tar.xz","size":342828,"last_modified":"2016-09-09 14:13","types":["bridge-server-descriptor 1.2"],"first_published":"2008-06-01 00:23","last_published":"2008-06-30 23:29","sha256":"+R6WdoPIjlGZQhkMEFhr5sOF1LOs2u/ioJyi3G8f9iI="},{"path":"bridge-server-descriptors-2008-07.tar.xz","size":374848,"last_modified":"2016-09-09 14:13","types":["bridge-server-descriptor 1.2"],"first_published":"2008-07-01 05:06","last_published":"2008-07-31 23:59","sha256":"/qR5Ec3Np/aKaHNGwSJJBxkFseKFaxYSrCNYeMqVS04="}]},{"path":"statuses","files":[{"path":"bridge-statuses-2008-05.tar.xz","size":74792,"last_modified":"2016-09-14 21:11","types":["bridge-netwo
 rk-status 1.1"],"first_published":"2008-05-16 19:46","last_published":"2008-05-31 23:37","sha256":"scynC2b8xKD+NbkejGK7mKCegUwGPwgzXu7MouxBSj0="},{"path":"bridge-statuses-2008-06.tar.xz","size":123488,"last_modified":"2016-09-14 21:11","types":["bridge-network-status 1.1"],"first_published":"2008-06-01 00:07","last_published":"2008-06-30 23:37","sha256":"sEEyUlnpL16UMfVIDlkFx/M5CRCa4A6wPAUUTN51eTA="},{"path":"bridge-statuses-2008-07.tar.xz","size":143836,"last_modified":"2016-09-14 21:11","types":["bridge-network-status 1.1"],"first_published":"2008-07-01 00:07","last_published":"2008-07-31 23:37","sha256":"cUIkQce1OEEgYHBpB+2AP2uKoRt+RoXOlkNKqPtacb8="}]}]},{"path":"bridge-pool-assignments","files":[{"path":"bridge-pool-assignments-2010-09.tar.xz","size":32804,"last_modified":"2012-05-31 10:21","types":["bridge-pool-assignment 1.0"],"first_published":"2010-09-29 07:41","last_published":"2010-09-30 23:37","sha256":"sBeIx6oEwkqjZkS9iB8Y9z/ArL8hdcGk9QMxt9dCUak="},{"path":"bridge-pool-a
 ssignments-2010-10.tar.xz","size":304684,"last_modified":"2012-05-31 10:21","types":["bridge-pool-assignment 1.0"],"first_published":"2010-10-01 00:07","last_published":"2010-10-31 23:37","sha256":"nQ3w/COujflQGGE/xTD9U9x6iMNZbbUf7QHde3siJTw="},{"path":"bridge-pool-assignments-2010-11.tar.xz","size":292228,"last_modified":"2012-05-31 10:21","types":["bridge-pool-assignment 1.0"],"first_published":"2010-11-01 00:07","last_published":"2010-11-30 23:37","sha256":"AxuzHdixo9j8AKJYXEUeoHiYL2/vaIbtb6tN7cYTfSg="}]},{"path":"bridgedb-metrics","files":[{"path":"bridgedb-metrics-2019-09.tar.xz","size":7728,"last_modified":"2019-11-10 15:27","first_published":"2019-09-05 08:51","last_published":"2019-09-30 14:51","sha256":"SJ6F54RXNfJBUPu9qq8zu2nDyXLKurhpZ2zyA1x+T6I="},{"path":"bridgedb-metrics-2019-10.tar.xz","size":9028,"last_modified":"2019-11-10 15:27","first_published":"2019-10-01 14:51","last_published":"2019-10-31 16:22","sha256":"pMrbEjNFQVy8F13EXThFIbOTHKggvahhbWuJ047QFPI="},{"path":"
 bridgedb-metrics-2019-11.tar.xz","size":17140,"last_modified":"2019-11-29 16:52","first_published":"2019-11-01 16:22","last_published":"2019-11-29 14:21","sha256":"x1GdLDMINdwm4pmYnvpLM8C91YwxwRzeGVVFcVW6B0o="}]},{"path":"exit-lists","files":[{"path":"exit-list-2010-02.tar.xz","size":272008,"last_modified":"2012-05-31 18:57","types":["tordnsel 1.0"],"first_published":"2010-02-22 15:32","last_published":"2010-02-28 23:18","sha256":"Q6ZAAy7RVbO+8rHH48AEZUU9PqcY5jD9zMASqjMzyns="},{"path":"exit-list-2010-03.tar.xz","size":1247484,"last_modified":"2012-05-31 18:57","types":["tordnsel 1.0"],"first_published":"2010-03-01 00:18","last_published":"2010-03-31 23:25","sha256":"ZjYIOYsuAwhmVYKs5QH7IQrRuoOicMy26vBxqnKBimY="},{"path":"exit-list-2010-04.tar.xz","size":1139896,"last_modified":"2012-05-31 18:57","types":["tordnsel 1.0"],"first_published":"2010-04-01 00:30","last_published":"2010-04-30 23:44","sha256":"gu8kB7FLJpoY+gYW/DsK2eR3hRYaAo+CrTn4hTVAEhM="}]},{"path":"relay-descriptors","file
 s":[{"path":"certs.tar.xz","size":151748,"last_modified":"2019-11-29 03:33","types":["dir-key-certificate-3 1.0"],"first_published":"2007-09-19 03:14","last_published":"2019-10-08 04:06","sha256":"ZfcE9RJwHvXhXaZ2xDzpoOJFqJeQR5ovOePOyNkKDi8="}],"directories":[{"path":"bandwidths","files":[{"path":"bandwidths-2017-08.tar.xz","size":13330020,"last_modified":"2019-07-29 18:45","types":["bandwidth-file 1.0"],"first_published":"2017-08-09 09:35","last_published":"2017-08-31 23:35","sha256":"BVVDEkegsLzkAn30dYikr4yTT79+XScfc0VUVEb83tM="},{"path":"bandwidths-2017-09.tar.xz","size":32672052,"last_modified":"2019-07-29 18:55","types":["bandwidth-file 1.0"],"first_published":"2017-09-01 00:40","last_published":"2017-09-30 23:38","sha256":"4Ify8gHvQ8HDVsNvol2cbOj3KPApOjbbANRp8OOqkyA="},{"path":"bandwidths-2017-10.tar.xz","size":40897092,"last_modified":"2019-07-29 19:18","types":["bandwidth-file 1.0"],"first_published":"2017-10-01 00:11","last_published":"2017-10-31 23:43","sha256":"Yy9Haap9QW
 vWN0E2z06T1KXHeEQmL3og9X9AWGN7mIc="}]},{"path":"consensuses","files":[{"path":"consensuses-2007-10.tar.xz","size":1061648,"last_modified":"2012-05-15 14:35","types":["network-status-consensus-3 1.0"],"first_published":"2007-10-27 12:00","last_published":"2007-10-31 23:00","sha256":"4lhYcEmZbJUcI5L4+p+Rt1/vn65L9b4mcdBQRzdTvKo="},{"path":"consensuses-2007-11.tar.xz","size":6810308,"last_modified":"2012-05-15 14:35","types":["network-status-consensus-3 1.0"],"first_published":"2007-11-01 00:00","last_published":"2007-11-30 23:00","sha256":"F3sbwfxsuvXclitWpO/rRniALV7EtEmKWFHf9LpT2ps="},{"path":"consensuses-2007-12.tar.xz","size":8106968,"last_modified":"2012-05-15 14:35","types":["network-status-consensus-3 1.0"],"first_published":"2007-12-01 00:00","last_published":"2007-12-31 23:00","sha256":"QIhi0QoI5bOJU2HawwqXp+oCBEASVyjFWAb72Ha5+0Q="}]},{"path":"extra-infos","files":[{"path":"extra-infos-2007-08.tar.xz","size":3016916,"last_modified":"2016-06-23 09:53","types":["extra-info 1.0"],
 "first_published":"2007-08-14 17:35","last_published":"2007-08-31 23:53","sha256":"UcAIrzYjFU52mRHXNle/fbI21lvfsVkeC0NpBZ/Pt/w="},{"path":"extra-infos-2007-09.tar.xz","size":6459884,"last_modified":"2016-06-23 09:54","types":["extra-info 1.0"],"first_published":"2007-09-01 00:04","last_published":"2007-09-30 23:57","sha256":"+hZ62VJrsIhkwqzZSbAQ3g1agPs+8QOgRVkcFxsIoBI="},{"path":"extra-infos-2007-10.tar.xz","size":7326564,"last_modified":"2016-06-23 09:54","types":["extra-info 1.0"],"first_published":"2007-10-01 00:01","last_published":"2007-10-31 23:59","sha256":"jLZGMaWik6X7zBJkHwoXSvplcwrNJhK2oRJTnBdM4MI="}]},{"path":"microdescs","files":[{"path":"microdescs-2014-01.tar.xz","size":7515396,"last_modified":"2014-02-07 03:59","types":["microdescriptor 1.0","network-status-microdesc-consensus-3 1.0"],"first_published":"2014-01-22 09:00","last_published":"2014-01-31 23:00","sha256":"DFugbV1phhpiEB0QeyyueKp0V/bicmAAkdBk/95RjKk="},{"path":"microdescs-2014-02.tar.xz","size":21822944,"las
 t_modified":"2014-03-07 04:54","types":["microdescriptor 1.0","network-status-microdesc-consensus-3 1.0"],"first_published":"2014-02-01 00:00","last_published":"2014-02-28 23:00","sha256":"5NFYLrbi8OUy7hRHhlWQ5uAbEkXDPHxkyFTkXIKiAF8="},{"path":"microdescs-2014-03.tar.xz","size":24201436,"last_modified":"2014-04-07 03:54","types":["microdescriptor 1.0","network-status-microdesc-consensus-3 1.0"],"first_published":"2014-03-01 00:00","last_published":"2014-03-31 23:00","sha256":"A6nUn1dyUx3s1sE5STBDH7+jW5Hdc2WD1mWBnSDmLVI="}]},{"path":"server-descriptors","files":[{"path":"server-descriptors-2005-12.tar.xz","size":1348620,"last_modified":"2016-06-24 08:12","types":["server-descriptor 1.0"],"first_published":"2005-12-15 01:42","last_published":"2005-12-17 11:06","sha256":"v3ANi2FD4xAhmyzigQq9gvlLwpXH8I6fGoiYlWLjOy8="},{"path":"server-descriptors-2006-02.tar.xz","size":28625408,"last_modified":"2016-06-24 08:14","types":["server-descriptor 1.0"],"first_published":"2006-02-04 11:22","last
 _published":"2006-02-28 23:59","sha256":"UhhE5TM/VM6VhhXnn7HdGMBw8L9RJL0NXsrZ0R/0DZQ="},{"path":"server-descriptors-2006-03.tar.xz","size":49548736,"last_modified":"2016-06-24 08:17","types":["server-descriptor 1.0"],"first_published":"2006-03-01 00:00","last_published":"2006-03-31 23:59","sha256":"Jv7qfAIbksfEjx80vW8aJto1cwTBDR1zD4LI2sNijXM="}]},{"path":"statuses","files":[{"path":"statuses-2005-12.tar.xz","size":1468844,"last_modified":"2016-06-25 11:50","types":["network-status-2 1.0"],"first_published":"2005-12-16 00:13","last_published":"2005-12-31 23:59","sha256":"Ec6DZrTog8DnFkmM7yRiAHO55lS9vOSDZrTtKHL0Vrw="},{"path":"statuses-2006-01.tar.xz","size":3344280,"last_modified":"2016-06-25 11:52","types":["network-status-2 1.0"],"first_published":"2006-01-01 00:38","last_published":"2006-01-31 23:39","sha256":"w3i+973xjkGh/qCvdM7i3bUq7zTDlWX9OXTUbaTe7ds="},{"path":"statuses-2006-02.tar.xz","size":4006336,"last_modified":"2016-06-25 11:54","types":["network-status-2 1.0"],"first_pu
 blished":"2006-02-01 00:37","last_published":"2006-02-28 23:38","sha256":"b4q9NCSDd2KMb0W0al1+LBQAGGu+IhLZIC7fONZCRC4="}]},{"path":"tor","files":[{"path":"tor-2004-05.tar.xz","size":386672,"last_modified":"2012-05-18 14:26","types":["directory 1.0"],"first_published":"2004-05-15 07:30","last_published":"2004-05-31 22:07","sha256":"0dpNzcmozHGhCzz6SnF4YSXUgSGPTeRLGPHJV4eCZjI="},{"path":"tor-2004-06.tar.xz","size":1087980,"last_modified":"2012-05-18 14:26","types":["directory 1.0"],"first_published":"2004-06-01 02:07","last_published":"2004-06-30 22:07","sha256":"qaXf+eht8EPlr9qO/VV9h+GjmxhboEP8BwLXw+Qn7+M="},{"path":"tor-2004-07.tar.xz","size":1366568,"last_modified":"2012-05-18 14:26","types":["directory 1.0"],"first_published":"2004-07-01 02:07","last_published":"2004-07-31 22:07","sha256":"f++lUmy43l+59UxH9e+Fxq/5MpnWvqRcIaJfNCKDwUE="}]},{"path":"votes","files":[{"path":"votes-2007-10.tar.xz","size":1356504,"last_modified":"2012-05-15 14:51","types":["network-status-vote-3 1.0"],"
 first_published":"2007-10-27 12:00","last_published":"2007-10-31 23:00","sha256":"j6e2TIb3LYCG3Bf50W9MFljiadWY7v/fe7K6+jnI+Hc="},{"path":"votes-2007-11.tar.xz","size":10641492,"last_modified":"2012-05-15 14:51","types":["network-status-vote-3 1.0"],"first_published":"2007-11-01 00:00","last_published":"2007-11-30 23:00","sha256":"mnb+jeueqm8WVbx8taipzE+yyzQ/rDuGaytCWvOS6NA="},{"path":"votes-2007-12.tar.xz","size":14712136,"last_modified":"2012-05-15 14:52","types":["network-status-vote-3 1.0"],"first_published":"2007-12-01 00:00","last_published":"2007-12-31 23:00","sha256":"tAjI4CWqtGpFNWb3DaIvzVugrPnHN2oyKAI1Z6ofjs0="}]}]},{"path":"snowflakes","files":[{"path":"snowflakes-2019-06.tar.xz","size":424,"last_modified":"2019-09-16 10:21","types":["snowflake-stats 1.0"],"first_published":"2019-06-29 21:41","last_published":"2019-06-30 21:41","sha256":"6o+UkFo5DuAju8J+w+dcyNEcAzhppueJOrAE4tKpHIw="},{"path":"snowflakes-2019-07.tar.xz","size":3528,"last_modified":"2019-09-16 10:21","types"
 :["snowflake-stats 1.0"],"first_published":"2019-07-01 21:41","last_published":"2019-07-31 19:52","sha256":"cd2eJpugSZuIpuSlQqw8Z0/YFjOuchMekD0uQQH/XMo="},{"path":"snowflakes-2019-08.tar.xz","size":6420,"last_modified":"2019-09-16 10:21","types":["snowflake-stats 1.0"],"first_published":"2019-08-01 19:52","last_published":"2019-08-31 14:06","sha256":"5k6Py4+ENKkcI0s4mpJWzcJ82XQvdl4t6wa8/Y+0sFI="}]},{"path":"torperf","files":[{"path":"torperf-2009-07.tar.xz","size":182712,"last_modified":"2012-05-30 07:23","types":["torperf 1.0"],"first_published":"2009-07-03 21:35","last_published":"2009-07-31 23:55","sha256":"mxi+0MgtNHXZ2wgYH/cre0go8yXn0TpRHJOA863KsWM="},{"path":"torperf-2009-08.tar.xz","size":203236,"last_modified":"2012-05-30 07:23","types":["torperf 1.0"],"first_published":"2009-08-01 00:00","last_published":"2009-08-31 23:55","sha256":"8xyc9zWnKUPoMO4mflyu2v5i2r2JifNNZ7noddfPGV0="},{"path":"torperf-2009-09.tar.xz","size":193832,"last_modified":"2012-05-30 07:23","types":["torp
 erf 1.0"],"first_published":"2009-09-01 00:00","last_published":"2009-09-30 23:55","sha256":"LCBvAkFgdpAgW71mmrVRNC/4oYKmu6ritc1xCBrQ6+o="}]},{"path":"webstats","files":[{"path":"webstats-2015-01.tar","size":30720,"last_modified":"2018-03-19 16:07","first_published":"2015-01-04 00:00","last_published":"2015-01-20 00:00","sha256":"TJb1iwzRBTyOvPuOL7DoKVUQa/Q5V0zD/7qNIS8QTp0="},{"path":"webstats-2015-02.tar","size":20480,"last_modified":"2018-03-19 16:07","first_published":"2015-02-11 00:00","last_published":"2015-02-12 00:00","sha256":"30uazpNDayxBRe7BEnfCRu8nrquDtoL/1oXV5eaX9xw="},{"path":"webstats-2015-03.tar","size":20480,"last_modified":"2018-03-19 16:07","first_published":"2015-03-04 00:00","last_published":"2015-03-13 00:00","sha256":"EPj4SrLYpnkfSUAOOfXeYBoE3zWP/VPq+bd8hhnOPKc="}]}]},{"path":"recent","directories":[{"path":"bridge-descriptors","directories":[{"path":"extra-infos","files":[{"path":"2019-11-27-23-09-00-extra-infos","size":753174,"last_modified":"2019-11-27 23:09
 ","types":["bridge-extra-info 1.3"],"first_published":"2019-11-27 21:58","last_published":"2019-11-28 01:05","sha256":"9wk9MvCL0Cu1SunJdWsuWXGh0EXe/hsOu03zYLA2BzQ="},{"path":"2019-11-28-00-09-00-extra-infos","size":793414,"last_modified":"2019-11-28 00:09","types":["bridge-extra-info 1.3"],"first_published":"2019-11-27 21:00","last_published":"2019-11-28 00:07","sha256":"AFM7LdUqT/zJHYsWZBOEEMUCAg1WvEaRExp3aa7mMy0="},{"path":"2019-11-28-01-09-00-extra-infos","size":616029,"last_modified":"2019-11-28 01:09","types":["bridge-extra-info 1.3"],"first_published":"2019-11-28 00:06","last_published":"2019-11-28 01:10","sha256":"ErLKI17AJHsQfkkckrz/dDkXUF57Olmoe6uAbxQZUYg="}]},{"path":"statuses","files":[{"path":"20191127-221129-BA44A889E64B93FAA2B114E02C2A279A8555C533","size":257731,"last_modified":"2019-11-27 23:09","types":["bridge-network-status 1.2"],"first_published":"2019-11-27 22:11","last_published":"2019-11-27 22:11","sha256":"fBty8+Okn2bEtTkpYm6Flz+j4U/8yHt5IVYl+T8AwAg="},{"path"
 :"20191127-224129-BA44A889E64B93FAA2B114E02C2A279A8555C533","size":257484,"last_modified":"2019-11-27 23:09","types":["bridge-network-status 1.2"],"first_published":"2019-11-27 22:41","last_published":"2019-11-27 22:41","sha256":"wkaV5zH/THEkYNU10PuOKWTAPjZrg1HMFVlX+nGtc7U="},{"path":"20191127-231129-BA44A889E64B93FAA2B114E02C2A279A8555C533","size":257459,"last_modified":"2019-11-28 00:09","types":["bridge-network-status 1.2"],"first_published":"2019-11-27 23:11","last_published":"2019-11-27 23:11","sha256":"7drOuyxG+xgZ0BMboAwbDkdHEZXiyt8yy1+j3nhGgQM="}]}]},{"path":"bridge-pool-assignments","files":[{"path":"2019-08-22-00-00-21","size":84863,"last_modified":"2019-11-28 10:09","types":["bridge-pool-assignment 1.0"],"first_published":"2019-08-22 00:00","last_published":"2019-08-22 00:00","sha256":"UX/6vleSKRBUP4fGnOPjUnpAZtFjkl/N4Tog3+A7KmU="},{"path":"2019-08-22-00-30-21","size":84363,"last_modified":"2019-11-28 10:09","types":["bridge-pool-assignment 1.0"],"first_published":"2019-0
 8-22 00:30","last_published":"2019-08-22 00:30","sha256":"8dH8DK5RuxUtDMJUA9JKkz+4vjPWbTK2Aqq4vvd03oM="},{"path":"2019-08-22-01-00-20","size":83912,"last_modified":"2019-11-28 10:09","types":["bridge-pool-assignment 1.0"],"first_published":"2019-08-22 01:00","last_published":"2019-08-22 01:00","sha256":"4JLrB0P4p14DrbNOXQTscT2dZESwgaZoszNWEiaYeTs="}]},{"path":"bridgedb-metrics","files":[{"path":"2019-09-05-08-51-16-bridgedb-metrics","size":7727,"last_modified":"2019-11-28 19:40","first_published":"2019-09-05 08:51","last_published":"2019-09-05 08:51","sha256":"gvgAeZ5ttvRmHFUmauZwA4scmCPCOhnBTjan/EPV1fk="},{"path":"2019-09-06-08-51-16-bridgedb-metrics","size":8134,"last_modified":"2019-11-28 19:40","first_published":"2019-09-06 08:51","last_published":"2019-09-06 08:51","sha256":"D/3QDwi7eRlkmGl/sk2uXmqn/6tJ3QLdAdArMt+a9kc="},{"path":"2019-09-07-08-51-16-bridgedb-metrics","size":7772,"last_modified":"2019-11-28 19:40","first_published":"2019-09-07 08:51","last_published":"2019-09-07
  08:51","sha256":"bMceCkg/6qFiRL7TKBsfWlQs1MoYVngavRFSDcxa8Gc="}]},{"path":"exit-lists","files":[{"path":"2019-11-27-23-02-00","size":160234,"last_modified":"2019-11-27 23:02","types":["tordnsel 1.0"],"first_published":"2019-11-27 23:02","last_published":"2019-11-27 23:02","sha256":"aOe9LSWJJ3N8sbembPFH8b027FlK2aixvKOkaDSEyUA="},{"path":"2019-11-28-00-02-00","size":160234,"last_modified":"2019-11-28 00:02","types":["tordnsel 1.0"],"first_published":"2019-11-28 00:02","last_published":"2019-11-28 00:02","sha256":"VuaqrB4wJid7RqHXn+BrL65F+FyKgMzzaj3XNumyOhc="}]},{"path":"relay-descriptors","directories":[{"path":"bandwidths","files":[{"path":"2019-11-27-20-44-46-bandwidth-572936DECB11EE615C0F698D0BB0E43D12F7A8B373F26236DBC4091099D3284A","size":2510298,"last_modified":"2019-11-27 22:35","types":["bandwidth-file 1.0"],"first_published":"2019-11-27 20:44","last_published":"2019-11-27 20:44","sha256":"Ystd7EP1ElJ/SSLUAMklGtbhvgO2Qu2+9SIemBLieO8="},{"path":"2019-11-27-21-31-40-bandwidth-3A
 522F869B89A2DDB0A798C3878483A0A3CC1E8F02FDF1C81404A5F742729C29","size":2450793,"last_modified":"2019-11-27 22:35","types":["bandwidth-file 1.0"],"first_published":"2019-11-27 21:31","last_published":"2019-11-27 21:31","sha256":"Wu1Fzp0qSXlNavxBAbm3TRj7AE5jGEdwrAYdoAhDMRE="},{"path":"2019-11-27-21-32-55-bandwidth-97EC4B3E17125BD9B58CE219FC417BA7B41C84D765711C8841AC75EB08CC5D8F","size":2510643,"last_modified":"2019-11-27 23:35","types":["bandwidth-file 1.0"],"first_published":"2019-11-27 21:32","last_published":"2019-11-27 21:32","sha256":"8VN/DJqQ0rmhfJvcr+yuwGSbvMDlkRfHyCnS5Y8GUuo="}]},{"path":"consensuses","files":[{"path":"2019-11-27-23-00-00-consensus","size":2208505,"last_modified":"2019-11-27 23:05","types":["network-status-consensus-3 1.0"],"first_published":"2019-11-27 23:00","last_published":"2019-11-27 23:00","sha256":"cGWT19Y0UVE/EUi3ZayacGvJU5t9T6MKaTOrNarAqlI="},{"path":"2019-11-28-00-00-00-consensus","size":2198681,"last_modified":"2019-11-28 00:05","types":["network-st
 atus-consensus-3 1.0"],"first_published":"2019-11-28 00:00","last_published":"2019-11-28 00:00","sha256":"VHwgrdBK3QpD7NQypBPqeFPXUjfFnuSYtmp/36jBUOo="},{"path":"2019-11-28-01-00-00-consensus","size":2195362,"last_modified":"2019-11-28 01:05","types":["network-status-consensus-3 1.0"],"first_published":"2019-11-28 01:00","last_published":"2019-11-28 01:00","sha256":"cXrIUvqiIybA87NoejHaBlDcN3VsLljGlmt+ku+HU/g="}]},{"path":"extra-infos","files":[{"path":"2019-11-27-23-05-00-extra-infos","size":1567496,"last_modified":"2019-11-27 23:05","types":["extra-info 1.0"],"first_published":"2019-11-27 21:00","last_published":"2019-11-27 23:28","sha256":"xNzjDxg6iCQFdfShUD4e2oO+SiLxSgVqQp13dXDjwTU="},{"path":"2019-11-28-00-05-00-extra-infos","size":1471455,"last_modified":"2019-11-28 00:05","types":["extra-info 1.0"],"first_published":"2019-11-27 19:04","last_published":"2019-11-28 00:01","sha256":"YybmmLOJgLf/kUyZ/IXQc5A4m7O8Sx0g6Ccu/G9ADhA="},{"path":"2019-11-28-01-05-00-extra-infos","size":1
 378360,"last_modified":"2019-11-28 01:05","types":["extra-info 1.0"],"first_published":"2019-11-27 10:31","last_published":"2019-11-28 01:17","sha256":"2nRmT48/8xVctXlhEHgawQVQy4DpcHS2NJHpQ1mZJRs="}]},{"path":"microdescs","directories":[{"path":"consensus-microdesc","files":[{"path":"2019-11-27-23-00-00-consensus-microdesc","size":2036635,"last_modified":"2019-11-27 23:05","types":["network-status-microdesc-consensus-3 1.0"],"first_published":"2019-11-27 23:00","last_published":"2019-11-27 23:00","sha256":"AEiZyuLlIf17aqIp3EOder5EzbSjgWesKaPLR/zRw6Q="},{"path":"2019-11-28-00-00-00-consensus-microdesc","size":2029885,"last_modified":"2019-11-28 00:05","types":["network-status-microdesc-consensus-3 1.0"],"first_published":"2019-11-28 00:00","last_published":"2019-11-28 00:00","sha256":"5P5AYHwuULmjNjdmubf0O6RYkIy7QAHLaIfv1GAUiYY="},{"path":"2019-11-28-01-00-00-consensus-microdesc","size":2026257,"last_modified":"2019-11-28 01:05","types":["network-status-microdesc-consensus-3 1.0"],"f
 irst_published":"2019-11-28 01:00","last_published":"2019-11-28 01:00","sha256":"jjqsrEr3kkT7P04BlZmWICrmUxMkUp+vmtNQl93OnJc="}]},{"path":"micro","files":[{"path":"2019-11-27-22-05-00-micro","size":14940,"last_modified":"2019-11-27 22:05","types":["microdescriptor 1.0"],"sha256":"cMrDxiUTrotiz80JaPQXEJ7UmNautHI+1tV7Jlvwn6Y="},{"path":"2019-11-27-23-05-00-micro","size":12234,"last_modified":"2019-11-27 23:05","types":["microdescriptor 1.0"],"sha256":"qzQ5EdHwdDNFAx6Ca7DgKdaABVznjmMMl+TNUMsKR1s="},{"path":"2019-11-28-00-05-00-micro","size":14696,"last_modified":"2019-11-28 00:05","types":["microdescriptor 1.0"],"sha256":"9mnZR7wYjdQm259cay9YgGzxoGSqdjn1sl3oODEggDA="}]}]},{"path":"server-descriptors","files":[{"path":"2019-11-27-23-05-00-server-descriptors","size":1978883,"last_modified":"2019-11-27 23:05","types":["server-descriptor 1.0"],"first_published":"2019-11-27 21:00","last_published":"2019-11-27 23:28","sha256":"HLb5s027ceH0iNag9IzQxR42pSMWreTxtC8xuNyNMs0="},{"path":"2019-11-2
 8-00-05-00-server-descriptors","size":1780252,"last_modified":"2019-11-28 00:05","types":["server-descriptor 1.0"],"first_published":"2019-11-27 19:04","last_published":"2019-11-28 00:01","sha256":"35HKfcpwZpOP6DU/6078dciDIfyt/7rbVfL6qOPphrM="},{"path":"2019-11-28-01-05-00-server-descriptors","size":1737656,"last_modified":"2019-11-28 01:05","types":["server-descriptor 1.0"],"first_published":"2019-11-27 10:31","last_published":"2019-11-28 01:17","sha256":"gzhTR8EVKPnki8xT0XHcw2WV+BVmWJ6RxQjjhyxZ8Ro="}]},{"path":"votes","files":[{"path":"2019-11-27-23-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-82EBEC114F0D555B5A44C13533235AFC68BE707D","size":3721766,"last_modified":"2019-11-27 23:05","types":["network-status-vote-3 1.0"],"first_published":"2019-11-27 23:00","last_published":"2019-11-27 23:00","sha256":"uBb0SCyaR+hRrJBoHpcgzDtdKzZD7bsqWVAf4ivYNao="},{"path":"2019-11-27-23-00-00-vote-14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4-1E73049822816A032C719782F7EBA461E1A43055","size":339
 6129,"last_modified":"2019-11-27 23:05","types":["network-status-vote-3 1.0"],"first_published":"2019-11-27 23:00","last_published":"2019-11-27 23:00","sha256":"WyUizNcJZIvKfJTc9cwUy1iyvSxfU5gAZGf3dSVQrMA="},{"path":"2019-11-27-23-00-00-vote-23D15D965BC35114467363C165C4F724B64B4F66-7D0503631CE3246ABD7A52BFAA4B416F02DDA5FA","size":3432263,"last_modified":"2019-11-27 23:05","types":["network-status-vote-3 1.0"],"first_published":"2019-11-27 23:00","last_published":"2019-11-27 23:00","sha256":"CIYCy5yb3FtJSpt8oDcqHUkuSnugtn6TnOvvR5XxGsk="}]}]},{"path":"snowflakes","files":[{"path":"2019-11-28-08-24-18-snowflake-stats","size":750,"last_modified":"2019-11-28 08:40","types":["snowflake-stats 1.0"],"first_published":"2019-11-28 08:24","last_published":"2019-11-28 08:24","sha256":"ikyXH8BpDQ03oZUw0P1+WyTLzoqQUma4mQ3ULJl74L4="},{"path":"2019-11-29-08-24-18-snowflake-stats","size":725,"last_modified":"2019-11-29 08:40","types":["snowflake-stats 1.0"],"first_published":"2019-11-29 08:24","last
 _published":"2019-11-29 08:24","sha256":"J5XTwjE1t/4mc7XLzr1mmYtoizIPH0WUdwNZs041nE0="},{"path":"2019-11-30-08-24-18-snowflake-stats","size":734,"last_modified":"2019-11-30 08:40","types":["snowflake-stats 1.0"],"first_published":"2019-11-30 08:24","last_published":"2019-11-30 08:24","sha256":"LyIeFYoZ+qnsXXZkLWkqXpqZ81eXorecQA6JOpi6kbw="}]},{"path":"torperf","files":[{"path":"op-ab-1048576-2019-11-27.tpf","size":42012,"last_modified":"2019-11-28 00:01","types":["torperf 1.1"],"first_published":"2019-11-27 00:19","last_published":"2019-11-27 23:59","sha256":"EIUcAWfl8hXt5Ho4r7UpNptBwJKC5VW+xtXDcsxIftI="},{"path":"op-ab-1048576-2019-11-28.tpf","size":39910,"last_modified":"2019-11-29 00:01","types":["torperf 1.1"],"first_published":"2019-11-28 00:24","last_published":"2019-11-28 23:59","sha256":"88mSSG4Vjl6Ue6TQ3ZggEmUE5N0/OZVnApJjhReegPg="},{"path":"op-ab-1048576-2019-11-29.tpf","size":47405,"last_modified":"2019-11-30 00:01","types":["torperf 1.1"],"first_published":"2019-11-29 00:
 04","last_published":"2019-11-29 22:44","sha256":"AdpYLfjx8dbyUgxg/wr/FJh0e/wNAuhffKp9rwdAzuk="}]},{"path":"webstats","files":[{"path":"2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20191116.xz","size":10480,"last_modified":"2019-11-28 19:41","first_published":"2019-11-16 00:00","last_published":"2019-11-16 00:00","sha256":"mJdqQBAfEPze/TuhLxBy9//1CFwCvbxHiIwwqptt1ek="},{"path":"2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20191117.xz","size":11836,"last_modified":"2019-11-30 19:41","first_published":"2019-11-17 00:00","last_published":"2019-11-17 00:00","sha256":"0vAFfbUWeoDn2kZyxK1a9CDZwTk5/yc9yz9IjEcPolo="},{"path":"2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20191118.xz","size":11108,"last_modified":"2019-11-30 19:41","first_published":"2019-11-18 00:00","last_published":"2019-11-18 00:00","sha256":"CcIuv30SczF0l6Z41d90+ykLobsRuqTfNT9TrfimcKk="}]}]}]}
diff --git a/test/unit/descriptor/data/collector/index.py b/test/unit/descriptor/data/collector/index.py
index 41ef6cbd..a68419db 100644
--- a/test/unit/descriptor/data/collector/index.py
+++ b/test/unit/descriptor/data/collector/index.py
@@ -1,8 +1,8 @@
 # far more readable python hash of collector_index.json
 
 EXAMPLE_INDEX = {
-  'index_created': '2019-07-06 01:54',
-  'build_revision': 'df98ac79',
+  'index_created': '2019-11-30 22:26',
+  'build_revision': '199d1bed',
   'path': 'https://collector.torproject.org',
   'directories': [
     {
@@ -16,16 +16,28 @@ EXAMPLE_INDEX = {
               'files': [
                 {
                   'path': 'bridge-extra-infos-2008-05.tar.xz',
-                  'size': 377644,
-                  'last_modified': '2016-09-04 09:21'
+                  'last_published': '2008-05-31 23:09',
+                  'first_published': '2008-05-13 15:21',
+                  'last_modified': '2016-09-04 09:21',
+                  'sha256': 'aDD2q7uNGOM+WuH67+nTd7rvFN4P580xPAmXYtqxr2I=',
+                  'types': ['bridge-extra-info 1.3'],
+                  'size': 377644
                 }, {
                   'path': 'bridge-extra-infos-2008-06.tar.xz',
-                  'size': 600484,
-                  'last_modified': '2016-09-04 09:21'
+                  'last_published': '2008-06-30 23:29',
+                  'first_published': '2008-06-01 00:23',
+                  'last_modified': '2016-09-04 09:21',
+                  'sha256': 'VLdUu98Fz8RKTFL13ftA3WJ5lsDJQBRrh/KGLOYAUqk=',
+                  'types': ['bridge-extra-info 1.3'],
+                  'size': 600484
                 }, {
                   'path': 'bridge-extra-infos-2008-07.tar.xz',
-                  'size': 716320,
-                  'last_modified': '2016-09-04 09:21'
+                  'last_published': '2008-07-31 23:59',
+                  'first_published': '2008-07-01 05:06',
+                  'last_modified': '2016-09-04 09:21',
+                  'sha256': 'NNj6mBr/aWDdNZmmPYSERn5pjQh0g04Xu8gUi6kMVds=',
+                  'types': ['bridge-extra-info 1.3'],
+                  'size': 716320
                 }
               ]
             }, {
@@ -33,16 +45,28 @@ EXAMPLE_INDEX = {
               'files': [
                 {
                   'path': 'bridge-server-descriptors-2008-05.tar.xz',
-                  'size': 205348,
-                  'last_modified': '2016-09-09 14:13'
+                  'last_published': '2008-05-31 23:09',
+                  'first_published': '2008-05-14 18:22',
+                  'last_modified': '2016-09-09 14:13',
+                  'sha256': 'NRb2dzS2OhFKYjfr5WoleOFokqC4C+qf0Nu4iegnFLo=',
+                  'types': ['bridge-server-descriptor 1.2'],
+                  'size': 205348
                 }, {
                   'path': 'bridge-server-descriptors-2008-06.tar.xz',
-                  'size': 342828,
-                  'last_modified': '2016-09-09 14:13'
+                  'last_published': '2008-06-30 23:29',
+                  'first_published': '2008-06-01 00:23',
+                  'last_modified': '2016-09-09 14:13',
+                  'sha256': '+R6WdoPIjlGZQhkMEFhr5sOF1LOs2u/ioJyi3G8f9iI=',
+                  'types': ['bridge-server-descriptor 1.2'],
+                  'size': 342828
                 }, {
                   'path': 'bridge-server-descriptors-2008-07.tar.xz',
-                  'size': 374848,
-                  'last_modified': '2016-09-09 14:13'
+                  'last_published': '2008-07-31 23:59',
+                  'first_published': '2008-07-01 05:06',
+                  'last_modified': '2016-09-09 14:13',
+                  'sha256': '/qR5Ec3Np/aKaHNGwSJJBxkFseKFaxYSrCNYeMqVS04=',
+                  'types': ['bridge-server-descriptor 1.2'],
+                  'size': 374848
                 }
               ]
             }, {
@@ -50,16 +74,28 @@ EXAMPLE_INDEX = {
               'files': [
                 {
                   'path': 'bridge-statuses-2008-05.tar.xz',
-                  'size': 74792,
-                  'last_modified': '2016-09-14 21:11'
+                  'last_published': '2008-05-31 23:37',
+                  'first_published': '2008-05-16 19:46',
+                  'last_modified': '2016-09-14 21:11',
+                  'sha256': 'scynC2b8xKD+NbkejGK7mKCegUwGPwgzXu7MouxBSj0=',
+                  'types': ['bridge-network-status 1.1'],
+                  'size': 74792
                 }, {
                   'path': 'bridge-statuses-2008-06.tar.xz',
-                  'size': 123488,
-                  'last_modified': '2016-09-14 21:11'
+                  'last_published': '2008-06-30 23:37',
+                  'first_published': '2008-06-01 00:07',
+                  'last_modified': '2016-09-14 21:11',
+                  'sha256': 'sEEyUlnpL16UMfVIDlkFx/M5CRCa4A6wPAUUTN51eTA=',
+                  'types': ['bridge-network-status 1.1'],
+                  'size': 123488
                 }, {
                   'path': 'bridge-statuses-2008-07.tar.xz',
-                  'size': 143836,
-                  'last_modified': '2016-09-14 21:11'
+                  'last_published': '2008-07-31 23:37',
+                  'first_published': '2008-07-01 00:07',
+                  'last_modified': '2016-09-14 21:11',
+                  'sha256': 'cUIkQce1OEEgYHBpB+2AP2uKoRt+RoXOlkNKqPtacb8=',
+                  'types': ['bridge-network-status 1.1'],
+                  'size': 143836
                 }
               ]
             }
@@ -69,16 +105,54 @@ EXAMPLE_INDEX = {
           'files': [
             {
               'path': 'bridge-pool-assignments-2010-09.tar.xz',
-              'size': 32804,
-              'last_modified': '2012-05-31 10:21'
+              'last_published': '2010-09-30 23:37',
+              'first_published': '2010-09-29 07:41',
+              'last_modified': '2012-05-31 10:21',
+              'sha256': 'sBeIx6oEwkqjZkS9iB8Y9z/ArL8hdcGk9QMxt9dCUak=',
+              'types': ['bridge-pool-assignment 1.0'],
+              'size': 32804
             }, {
               'path': 'bridge-pool-assignments-2010-10.tar.xz',
-              'size': 304684,
-              'last_modified': '2012-05-31 10:21'
+              'last_published': '2010-10-31 23:37',
+              'first_published': '2010-10-01 00:07',
+              'last_modified': '2012-05-31 10:21',
+              'sha256': 'nQ3w/COujflQGGE/xTD9U9x6iMNZbbUf7QHde3siJTw=',
+              'types': ['bridge-pool-assignment 1.0'],
+              'size': 304684
             }, {
               'path': 'bridge-pool-assignments-2010-11.tar.xz',
-              'size': 292228,
-              'last_modified': '2012-05-31 10:21'
+              'last_published': '2010-11-30 23:37',
+              'first_published': '2010-11-01 00:07',
+              'last_modified': '2012-05-31 10:21',
+              'sha256': 'AxuzHdixo9j8AKJYXEUeoHiYL2/vaIbtb6tN7cYTfSg=',
+              'types': ['bridge-pool-assignment 1.0'],
+              'size': 292228
+            }
+          ]
+        }, {
+          'path': 'bridgedb-metrics',
+          'files': [
+            {
+              'path': 'bridgedb-metrics-2019-09.tar.xz',
+              'last_published': '2019-09-30 14:51',
+              'first_published': '2019-09-05 08:51',
+              'last_modified': '2019-11-10 15:27',
+              'sha256': 'SJ6F54RXNfJBUPu9qq8zu2nDyXLKurhpZ2zyA1x+T6I=',
+              'size': 7728
+            }, {
+              'path': 'bridgedb-metrics-2019-10.tar.xz',
+              'last_published': '2019-10-31 16:22',
+              'first_published': '2019-10-01 14:51',
+              'last_modified': '2019-11-10 15:27',
+              'sha256': 'pMrbEjNFQVy8F13EXThFIbOTHKggvahhbWuJ047QFPI=',
+              'size': 9028
+            }, {
+              'path': 'bridgedb-metrics-2019-11.tar.xz',
+              'last_published': '2019-11-29 14:21',
+              'first_published': '2019-11-01 16:22',
+              'last_modified': '2019-11-29 16:52',
+              'sha256': 'x1GdLDMINdwm4pmYnvpLM8C91YwxwRzeGVVFcVW6B0o=',
+              'size': 17140
             }
           ]
         }, {
@@ -86,16 +160,28 @@ EXAMPLE_INDEX = {
           'files': [
             {
               'path': 'exit-list-2010-02.tar.xz',
-              'size': 272008,
-              'last_modified': '2012-05-31 18:57'
+              'last_published': '2010-02-28 23:18',
+              'first_published': '2010-02-22 15:32',
+              'last_modified': '2012-05-31 18:57',
+              'sha256': 'Q6ZAAy7RVbO+8rHH48AEZUU9PqcY5jD9zMASqjMzyns=',
+              'types': ['tordnsel 1.0'],
+              'size': 272008
             }, {
               'path': 'exit-list-2010-03.tar.xz',
-              'size': 1247484,
-              'last_modified': '2012-05-31 18:57'
+              'last_published': '2010-03-31 23:25',
+              'first_published': '2010-03-01 00:18',
+              'last_modified': '2012-05-31 18:57',
+              'sha256': 'ZjYIOYsuAwhmVYKs5QH7IQrRuoOicMy26vBxqnKBimY=',
+              'types': ['tordnsel 1.0'],
+              'size': 1247484
             }, {
               'path': 'exit-list-2010-04.tar.xz',
-              'size': 1139896,
-              'last_modified': '2012-05-31 18:57'
+              'last_published': '2010-04-30 23:44',
+              'first_published': '2010-04-01 00:30',
+              'last_modified': '2012-05-31 18:57',
+              'sha256': 'gu8kB7FLJpoY+gYW/DsK2eR3hRYaAo+CrTn4hTVAEhM=',
+              'types': ['tordnsel 1.0'],
+              'size': 1139896
             }
           ]
         }, {
@@ -103,8 +189,12 @@ EXAMPLE_INDEX = {
           'files': [
             {
               'path': 'certs.tar.xz',
-              'size': 144696,
-              'last_modified': '2019-07-03 03:29'
+              'last_published': '2019-10-08 04:06',
+              'first_published': '2007-09-19 03:14',
+              'last_modified': '2019-11-29 03:33',
+              'sha256': 'ZfcE9RJwHvXhXaZ2xDzpoOJFqJeQR5ovOePOyNkKDi8=',
+              'types': ['dir-key-certificate-3 1.0'],
+              'size': 151748
             }
           ],
           'directories': [
@@ -112,17 +202,29 @@ EXAMPLE_INDEX = {
               'path': 'bandwidths',
               'files': [
                 {
-                  'path': 'bandwidths-2019-05.tar.xz',
-                  'size': 50385816,
-                  'last_modified': '2019-06-07 08:05'
-                }, {
-                  'path': 'bandwidths-2019-06.tar.xz',
-                  'size': 105881156,
-                  'last_modified': '2019-07-03 07:30'
-                }, {
-                  'path': 'bandwidths-2019-07.tar.xz',
-                  'size': 11374436,
-                  'last_modified': '2019-07-03 07:12'
+                  'path': 'bandwidths-2017-08.tar.xz',
+                  'last_published': '2017-08-31 23:35',
+                  'first_published': '2017-08-09 09:35',
+                  'last_modified': '2019-07-29 18:45',
+                  'sha256': 'BVVDEkegsLzkAn30dYikr4yTT79+XScfc0VUVEb83tM=',
+                  'types': ['bandwidth-file 1.0'],
+                  'size': 13330020
+                }, {
+                  'path': 'bandwidths-2017-09.tar.xz',
+                  'last_published': '2017-09-30 23:38',
+                  'first_published': '2017-09-01 00:40',
+                  'last_modified': '2019-07-29 18:55',
+                  'sha256': '4Ify8gHvQ8HDVsNvol2cbOj3KPApOjbbANRp8OOqkyA=',
+                  'types': ['bandwidth-file 1.0'],
+                  'size': 32672052
+                }, {
+                  'path': 'bandwidths-2017-10.tar.xz',
+                  'last_published': '2017-10-31 23:43',
+                  'first_published': '2017-10-01 00:11',
+                  'last_modified': '2019-07-29 19:18',
+                  'sha256': 'Yy9Haap9QWvWN0E2z06T1KXHeEQmL3og9X9AWGN7mIc=',
+                  'types': ['bandwidth-file 1.0'],
+                  'size': 40897092
                 }
               ]
             }, {
@@ -130,16 +232,28 @@ EXAMPLE_INDEX = {
               'files': [
                 {
                   'path': 'consensuses-2007-10.tar.xz',
-                  'size': 1061648,
-                  'last_modified': '2012-05-15 14:35'
+                  'last_published': '2007-10-31 23:00',
+                  'first_published': '2007-10-27 12:00',
+                  'last_modified': '2012-05-15 14:35',
+                  'sha256': '4lhYcEmZbJUcI5L4+p+Rt1/vn65L9b4mcdBQRzdTvKo=',
+                  'types': ['network-status-consensus-3 1.0'],
+                  'size': 1061648
                 }, {
                   'path': 'consensuses-2007-11.tar.xz',
-                  'size': 6810308,
-                  'last_modified': '2012-05-15 14:35'
+                  'last_published': '2007-11-30 23:00',
+                  'first_published': '2007-11-01 00:00',
+                  'last_modified': '2012-05-15 14:35',
+                  'sha256': 'F3sbwfxsuvXclitWpO/rRniALV7EtEmKWFHf9LpT2ps=',
+                  'types': ['network-status-consensus-3 1.0'],
+                  'size': 6810308
                 }, {
                   'path': 'consensuses-2007-12.tar.xz',
-                  'size': 8106968,
-                  'last_modified': '2012-05-15 14:35'
+                  'last_published': '2007-12-31 23:00',
+                  'first_published': '2007-12-01 00:00',
+                  'last_modified': '2012-05-15 14:35',
+                  'sha256': 'QIhi0QoI5bOJU2HawwqXp+oCBEASVyjFWAb72Ha5+0Q=',
+                  'types': ['network-status-consensus-3 1.0'],
+                  'size': 8106968
                 }
               ]
             }, {
@@ -147,16 +261,28 @@ EXAMPLE_INDEX = {
               'files': [
                 {
                   'path': 'extra-infos-2007-08.tar.xz',
-                  'size': 3016916,
-                  'last_modified': '2016-06-23 09:53'
+                  'last_published': '2007-08-31 23:53',
+                  'first_published': '2007-08-14 17:35',
+                  'last_modified': '2016-06-23 09:53',
+                  'sha256': 'UcAIrzYjFU52mRHXNle/fbI21lvfsVkeC0NpBZ/Pt/w=',
+                  'types': ['extra-info 1.0'],
+                  'size': 3016916
                 }, {
                   'path': 'extra-infos-2007-09.tar.xz',
-                  'size': 6459884,
-                  'last_modified': '2016-06-23 09:54'
+                  'last_published': '2007-09-30 23:57',
+                  'first_published': '2007-09-01 00:04',
+                  'last_modified': '2016-06-23 09:54',
+                  'sha256': '+hZ62VJrsIhkwqzZSbAQ3g1agPs+8QOgRVkcFxsIoBI=',
+                  'types': ['extra-info 1.0'],
+                  'size': 6459884
                 }, {
                   'path': 'extra-infos-2007-10.tar.xz',
-                  'size': 7326564,
-                  'last_modified': '2016-06-23 09:54'
+                  'last_published': '2007-10-31 23:59',
+                  'first_published': '2007-10-01 00:01',
+                  'last_modified': '2016-06-23 09:54',
+                  'sha256': 'jLZGMaWik6X7zBJkHwoXSvplcwrNJhK2oRJTnBdM4MI=',
+                  'types': ['extra-info 1.0'],
+                  'size': 7326564
                 }
               ]
             }, {
@@ -164,16 +290,28 @@ EXAMPLE_INDEX = {
               'files': [
                 {
                   'path': 'microdescs-2014-01.tar.xz',
-                  'size': 7515396,
-                  'last_modified': '2014-02-07 03:59'
+                  'last_published': '2014-01-31 23:00',
+                  'first_published': '2014-01-22 09:00',
+                  'last_modified': '2014-02-07 03:59',
+                  'sha256': 'DFugbV1phhpiEB0QeyyueKp0V/bicmAAkdBk/95RjKk=',
+                  'types': ['microdescriptor 1.0', 'network-status-microdesc-consensus-3 1.0'],
+                  'size': 7515396
                 }, {
                   'path': 'microdescs-2014-02.tar.xz',
-                  'size': 21822944,
-                  'last_modified': '2014-03-07 04:54'
+                  'last_published': '2014-02-28 23:00',
+                  'first_published': '2014-02-01 00:00',
+                  'last_modified': '2014-03-07 04:54',
+                  'sha256': '5NFYLrbi8OUy7hRHhlWQ5uAbEkXDPHxkyFTkXIKiAF8=',
+                  'types': ['microdescriptor 1.0', 'network-status-microdesc-consensus-3 1.0'],
+                  'size': 21822944
                 }, {
                   'path': 'microdescs-2014-03.tar.xz',
-                  'size': 24201436,
-                  'last_modified': '2014-04-07 03:54'
+                  'last_published': '2014-03-31 23:00',
+                  'first_published': '2014-03-01 00:00',
+                  'last_modified': '2014-04-07 03:54',
+                  'sha256': 'A6nUn1dyUx3s1sE5STBDH7+jW5Hdc2WD1mWBnSDmLVI=',
+                  'types': ['microdescriptor 1.0', 'network-status-microdesc-consensus-3 1.0'],
+                  'size': 24201436
                 }
               ]
             }, {
@@ -181,16 +319,28 @@ EXAMPLE_INDEX = {
               'files': [
                 {
                   'path': 'server-descriptors-2005-12.tar.xz',
-                  'size': 1348620,
-                  'last_modified': '2016-06-24 08:12'
+                  'last_published': '2005-12-17 11:06',
+                  'first_published': '2005-12-15 01:42',
+                  'last_modified': '2016-06-24 08:12',
+                  'sha256': 'v3ANi2FD4xAhmyzigQq9gvlLwpXH8I6fGoiYlWLjOy8=',
+                  'types': ['server-descriptor 1.0'],
+                  'size': 1348620
                 }, {
                   'path': 'server-descriptors-2006-02.tar.xz',
-                  'size': 28625408,
-                  'last_modified': '2016-06-24 08:14'
+                  'last_published': '2006-02-28 23:59',
+                  'first_published': '2006-02-04 11:22',
+                  'last_modified': '2016-06-24 08:14',
+                  'sha256': 'UhhE5TM/VM6VhhXnn7HdGMBw8L9RJL0NXsrZ0R/0DZQ=',
+                  'types': ['server-descriptor 1.0'],
+                  'size': 28625408
                 }, {
                   'path': 'server-descriptors-2006-03.tar.xz',
-                  'size': 49548736,
-                  'last_modified': '2016-06-24 08:17'
+                  'last_published': '2006-03-31 23:59',
+                  'first_published': '2006-03-01 00:00',
+                  'last_modified': '2016-06-24 08:17',
+                  'sha256': 'Jv7qfAIbksfEjx80vW8aJto1cwTBDR1zD4LI2sNijXM=',
+                  'types': ['server-descriptor 1.0'],
+                  'size': 49548736
                 }
               ]
             }, {
@@ -198,16 +348,28 @@ EXAMPLE_INDEX = {
               'files': [
                 {
                   'path': 'statuses-2005-12.tar.xz',
-                  'size': 1468844,
-                  'last_modified': '2016-06-25 11:50'
+                  'last_published': '2005-12-31 23:59',
+                  'first_published': '2005-12-16 00:13',
+                  'last_modified': '2016-06-25 11:50',
+                  'sha256': 'Ec6DZrTog8DnFkmM7yRiAHO55lS9vOSDZrTtKHL0Vrw=',
+                  'types': ['network-status-2 1.0'],
+                  'size': 1468844
                 }, {
                   'path': 'statuses-2006-01.tar.xz',
-                  'size': 3344280,
-                  'last_modified': '2016-06-25 11:52'
+                  'last_published': '2006-01-31 23:39',
+                  'first_published': '2006-01-01 00:38',
+                  'last_modified': '2016-06-25 11:52',
+                  'sha256': 'w3i+973xjkGh/qCvdM7i3bUq7zTDlWX9OXTUbaTe7ds=',
+                  'types': ['network-status-2 1.0'],
+                  'size': 3344280
                 }, {
                   'path': 'statuses-2006-02.tar.xz',
-                  'size': 4006336,
-                  'last_modified': '2016-06-25 11:54'
+                  'last_published': '2006-02-28 23:38',
+                  'first_published': '2006-02-01 00:37',
+                  'last_modified': '2016-06-25 11:54',
+                  'sha256': 'b4q9NCSDd2KMb0W0al1+LBQAGGu+IhLZIC7fONZCRC4=',
+                  'types': ['network-status-2 1.0'],
+                  'size': 4006336
                 }
               ]
             }, {
@@ -215,16 +377,28 @@ EXAMPLE_INDEX = {
               'files': [
                 {
                   'path': 'tor-2004-05.tar.xz',
-                  'size': 386672,
-                  'last_modified': '2012-05-18 14:26'
+                  'last_published': '2004-05-31 22:07',
+                  'first_published': '2004-05-15 07:30',
+                  'last_modified': '2012-05-18 14:26',
+                  'sha256': '0dpNzcmozHGhCzz6SnF4YSXUgSGPTeRLGPHJV4eCZjI=',
+                  'types': ['directory 1.0'],
+                  'size': 386672
                 }, {
                   'path': 'tor-2004-06.tar.xz',
-                  'size': 1087980,
-                  'last_modified': '2012-05-18 14:26'
+                  'last_published': '2004-06-30 22:07',
+                  'first_published': '2004-06-01 02:07',
+                  'last_modified': '2012-05-18 14:26',
+                  'sha256': 'qaXf+eht8EPlr9qO/VV9h+GjmxhboEP8BwLXw+Qn7+M=',
+                  'types': ['directory 1.0'],
+                  'size': 1087980
                 }, {
                   'path': 'tor-2004-07.tar.xz',
-                  'size': 1366568,
-                  'last_modified': '2012-05-18 14:26'
+                  'last_published': '2004-07-31 22:07',
+                  'first_published': '2004-07-01 02:07',
+                  'last_modified': '2012-05-18 14:26',
+                  'sha256': 'f++lUmy43l+59UxH9e+Fxq/5MpnWvqRcIaJfNCKDwUE=',
+                  'types': ['directory 1.0'],
+                  'size': 1366568
                 }
               ]
             }, {
@@ -232,35 +406,88 @@ EXAMPLE_INDEX = {
               'files': [
                 {
                   'path': 'votes-2007-10.tar.xz',
-                  'size': 1356504,
-                  'last_modified': '2012-05-15 14:51'
+                  'last_published': '2007-10-31 23:00',
+                  'first_published': '2007-10-27 12:00',
+                  'last_modified': '2012-05-15 14:51',
+                  'sha256': 'j6e2TIb3LYCG3Bf50W9MFljiadWY7v/fe7K6+jnI+Hc=',
+                  'types': ['network-status-vote-3 1.0'],
+                  'size': 1356504
                 }, {
                   'path': 'votes-2007-11.tar.xz',
-                  'size': 10641492,
-                  'last_modified': '2012-05-15 14:51'
+                  'last_published': '2007-11-30 23:00',
+                  'first_published': '2007-11-01 00:00',
+                  'last_modified': '2012-05-15 14:51',
+                  'sha256': 'mnb+jeueqm8WVbx8taipzE+yyzQ/rDuGaytCWvOS6NA=',
+                  'types': ['network-status-vote-3 1.0'],
+                  'size': 10641492
                 }, {
                   'path': 'votes-2007-12.tar.xz',
-                  'size': 14712136,
-                  'last_modified': '2012-05-15 14:52'
+                  'last_published': '2007-12-31 23:00',
+                  'first_published': '2007-12-01 00:00',
+                  'last_modified': '2012-05-15 14:52',
+                  'sha256': 'tAjI4CWqtGpFNWb3DaIvzVugrPnHN2oyKAI1Z6ofjs0=',
+                  'types': ['network-status-vote-3 1.0'],
+                  'size': 14712136
                 }
               ]
             }
           ]
         }, {
+          'path': 'snowflakes',
+          'files': [
+            {
+              'path': 'snowflakes-2019-06.tar.xz',
+              'last_published': '2019-06-30 21:41',
+              'first_published': '2019-06-29 21:41',
+              'last_modified': '2019-09-16 10:21',
+              'sha256': '6o+UkFo5DuAju8J+w+dcyNEcAzhppueJOrAE4tKpHIw=',
+              'types': ['snowflake-stats 1.0'],
+              'size': 424
+            }, {
+              'path': 'snowflakes-2019-07.tar.xz',
+              'last_published': '2019-07-31 19:52',
+              'first_published': '2019-07-01 21:41',
+              'last_modified': '2019-09-16 10:21',
+              'sha256': 'cd2eJpugSZuIpuSlQqw8Z0/YFjOuchMekD0uQQH/XMo=',
+              'types': ['snowflake-stats 1.0'],
+              'size': 3528
+            }, {
+              'path': 'snowflakes-2019-08.tar.xz',
+              'last_published': '2019-08-31 14:06',
+              'first_published': '2019-08-01 19:52',
+              'last_modified': '2019-09-16 10:21',
+              'sha256': '5k6Py4+ENKkcI0s4mpJWzcJ82XQvdl4t6wa8/Y+0sFI=',
+              'types': ['snowflake-stats 1.0'],
+              'size': 6420
+            }
+          ]
+        }, {
           'path': 'torperf',
           'files': [
             {
               'path': 'torperf-2009-07.tar.xz',
-              'size': 182712,
-              'last_modified': '2012-05-30 07:23'
+              'last_published': '2009-07-31 23:55',
+              'first_published': '2009-07-03 21:35',
+              'last_modified': '2012-05-30 07:23',
+              'sha256': 'mxi+0MgtNHXZ2wgYH/cre0go8yXn0TpRHJOA863KsWM=',
+              'types': ['torperf 1.0'],
+              'size': 182712
             }, {
               'path': 'torperf-2009-08.tar.xz',
-              'size': 203236,
-              'last_modified': '2012-05-30 07:23'
+              'last_published': '2009-08-31 23:55',
+              'first_published': '2009-08-01 00:00',
+              'last_modified': '2012-05-30 07:23',
+              'sha256': '8xyc9zWnKUPoMO4mflyu2v5i2r2JifNNZ7noddfPGV0=',
+              'types': ['torperf 1.0'],
+              'size': 203236
             }, {
               'path': 'torperf-2009-09.tar.xz',
-              'size': 193832,
-              'last_modified': '2012-05-30 07:23'
+              'last_published': '2009-09-30 23:55',
+              'first_published': '2009-09-01 00:00',
+              'last_modified': '2012-05-30 07:23',
+              'sha256': 'LCBvAkFgdpAgW71mmrVRNC/4oYKmu6ritc1xCBrQ6+o=',
+              'types': ['torperf 1.0'],
+              'size': 193832
             }
           ]
         }, {
@@ -268,23 +495,30 @@ EXAMPLE_INDEX = {
           'files': [
             {
               'path': 'webstats-2015-01.tar',
-              'size': 30720,
-              'last_modified': '2018-03-19 16:07'
+              'last_published': '2015-01-20 00:00',
+              'first_published': '2015-01-04 00:00',
+              'last_modified': '2018-03-19 16:07',
+              'sha256': 'TJb1iwzRBTyOvPuOL7DoKVUQa/Q5V0zD/7qNIS8QTp0=',
+              'size': 30720
             }, {
               'path': 'webstats-2015-02.tar',
-              'size': 20480,
-              'last_modified': '2018-03-19 16:07'
+              'last_published': '2015-02-12 00:00',
+              'first_published': '2015-02-11 00:00',
+              'last_modified': '2018-03-19 16:07',
+              'sha256': '30uazpNDayxBRe7BEnfCRu8nrquDtoL/1oXV5eaX9xw=',
+              'size': 20480
             }, {
               'path': 'webstats-2015-03.tar',
-              'size': 20480,
-              'last_modified': '2018-03-19 16:07'
+              'last_published': '2015-03-13 00:00',
+              'first_published': '2015-03-04 00:00',
+              'last_modified': '2018-03-19 16:07',
+              'sha256': 'EPj4SrLYpnkfSUAOOfXeYBoE3zWP/VPq+bd8hhnOPKc=',
+              'size': 20480
             }
           ]
         }
       ]
     }, {
-      'path': 'contrib'
-    }, {
       'path': 'recent',
       'directories': [
         {
@@ -294,70 +528,136 @@ EXAMPLE_INDEX = {
               'path': 'extra-infos',
               'files': [
                 {
-                  'path': '2019-07-03-02-09-00-extra-infos',
-                  'size': 507816,
-                  'last_modified': '2019-07-03 02:09'
-                }, {
-                  'path': '2019-07-03-03-09-00-extra-infos',
-                  'size': 558790,
-                  'last_modified': '2019-07-03 03:09'
-                }, {
-                  'path': '2019-07-03-04-09-05-extra-infos',
-                  'size': 592279,
-                  'last_modified': '2019-07-03 04:09'
-                }
-              ]
-            }, {
-              'path': 'server-descriptors',
-              'files': [
-                {
-                  'path': '2019-07-03-02-09-00-server-descriptors',
-                  'size': 274355,
-                  'last_modified': '2019-07-03 02:09'
-                }, {
-                  'path': '2019-07-03-03-09-00-server-descriptors',
-                  'size': 295671,
-                  'last_modified': '2019-07-03 03:09'
-                }, {
-                  'path': '2019-07-03-04-09-05-server-descriptors',
-                  'size': 312822,
-                  'last_modified': '2019-07-03 04:09'
+                  'path': '2019-11-27-23-09-00-extra-infos',
+                  'last_published': '2019-11-28 01:05',
+                  'first_published': '2019-11-27 21:58',
+                  'last_modified': '2019-11-27 23:09',
+                  'sha256': '9wk9MvCL0Cu1SunJdWsuWXGh0EXe/hsOu03zYLA2BzQ=',
+                  'types': ['bridge-extra-info 1.3'],
+                  'size': 753174
+                }, {
+                  'path': '2019-11-28-00-09-00-extra-infos',
+                  'last_published': '2019-11-28 00:07',
+                  'first_published': '2019-11-27 21:00',
+                  'last_modified': '2019-11-28 00:09',
+                  'sha256': 'AFM7LdUqT/zJHYsWZBOEEMUCAg1WvEaRExp3aa7mMy0=',
+                  'types': ['bridge-extra-info 1.3'],
+                  'size': 793414
+                }, {
+                  'path': '2019-11-28-01-09-00-extra-infos',
+                  'last_published': '2019-11-28 01:10',
+                  'first_published': '2019-11-28 00:06',
+                  'last_modified': '2019-11-28 01:09',
+                  'sha256': 'ErLKI17AJHsQfkkckrz/dDkXUF57Olmoe6uAbxQZUYg=',
+                  'types': ['bridge-extra-info 1.3'],
+                  'size': 616029
                 }
               ]
             }, {
               'path': 'statuses',
               'files': [
                 {
-                  'path': '20190703-011231-BA44A889E64B93FAA2B114E02C2A279A8555C533',
-                  'size': 238340,
-                  'last_modified': '2019-07-03 02:09'
-                }, {
-                  'path': '20190703-014231-BA44A889E64B93FAA2B114E02C2A279A8555C533',
-                  'size': 238147,
-                  'last_modified': '2019-07-03 02:09'
-                }, {
-                  'path': '20190703-021231-BA44A889E64B93FAA2B114E02C2A279A8555C533',
-                  'size': 238231,
-                  'last_modified': '2019-07-03 03:09'
+                  'path': '20191127-221129-BA44A889E64B93FAA2B114E02C2A279A8555C533',
+                  'last_published': '2019-11-27 22:11',
+                  'first_published': '2019-11-27 22:11',
+                  'last_modified': '2019-11-27 23:09',
+                  'sha256': 'fBty8+Okn2bEtTkpYm6Flz+j4U/8yHt5IVYl+T8AwAg=',
+                  'types': ['bridge-network-status 1.2'],
+                  'size': 257731
+                }, {
+                  'path': '20191127-224129-BA44A889E64B93FAA2B114E02C2A279A8555C533',
+                  'last_published': '2019-11-27 22:41',
+                  'first_published': '2019-11-27 22:41',
+                  'last_modified': '2019-11-27 23:09',
+                  'sha256': 'wkaV5zH/THEkYNU10PuOKWTAPjZrg1HMFVlX+nGtc7U=',
+                  'types': ['bridge-network-status 1.2'],
+                  'size': 257484
+                }, {
+                  'path': '20191127-231129-BA44A889E64B93FAA2B114E02C2A279A8555C533',
+                  'last_published': '2019-11-27 23:11',
+                  'first_published': '2019-11-27 23:11',
+                  'last_modified': '2019-11-28 00:09',
+                  'sha256': '7drOuyxG+xgZ0BMboAwbDkdHEZXiyt8yy1+j3nhGgQM=',
+                  'types': ['bridge-network-status 1.2'],
+                  'size': 257459
                 }
               ]
             }
           ]
         }, {
-          'path': 'exit-lists',
+          'path': 'bridge-pool-assignments',
           'files': [
             {
-              'path': '2019-07-03-02-02-00',
-              'size': 158516,
-              'last_modified': '2019-07-03 02:02'
+              'path': '2019-08-22-00-00-21',
+              'last_published': '2019-08-22 00:00',
+              'first_published': '2019-08-22 00:00',
+              'last_modified': '2019-11-28 10:09',
+              'sha256': 'UX/6vleSKRBUP4fGnOPjUnpAZtFjkl/N4Tog3+A7KmU=',
+              'types': ['bridge-pool-assignment 1.0'],
+              'size': 84863
             }, {
-              'path': '2019-07-03-03-02-00',
-              'size': 158830,
-              'last_modified': '2019-07-03 03:02'
+              'path': '2019-08-22-00-30-21',
+              'last_published': '2019-08-22 00:30',
+              'first_published': '2019-08-22 00:30',
+              'last_modified': '2019-11-28 10:09',
+              'sha256': '8dH8DK5RuxUtDMJUA9JKkz+4vjPWbTK2Aqq4vvd03oM=',
+              'types': ['bridge-pool-assignment 1.0'],
+              'size': 84363
             }, {
-              'path': '2019-07-03-04-02-00',
-              'size': 158831,
-              'last_modified': '2019-07-03 04:02'
+              'path': '2019-08-22-01-00-20',
+              'last_published': '2019-08-22 01:00',
+              'first_published': '2019-08-22 01:00',
+              'last_modified': '2019-11-28 10:09',
+              'sha256': '4JLrB0P4p14DrbNOXQTscT2dZESwgaZoszNWEiaYeTs=',
+              'types': ['bridge-pool-assignment 1.0'],
+              'size': 83912
+            }
+          ]
+        }, {
+          'path': 'bridgedb-metrics',
+          'files': [
+            {
+              'path': '2019-09-05-08-51-16-bridgedb-metrics',
+              'last_published': '2019-09-05 08:51',
+              'first_published': '2019-09-05 08:51',
+              'last_modified': '2019-11-28 19:40',
+              'sha256': 'gvgAeZ5ttvRmHFUmauZwA4scmCPCOhnBTjan/EPV1fk=',
+              'size': 7727
+            }, {
+              'path': '2019-09-06-08-51-16-bridgedb-metrics',
+              'last_published': '2019-09-06 08:51',
+              'first_published': '2019-09-06 08:51',
+              'last_modified': '2019-11-28 19:40',
+              'sha256': 'D/3QDwi7eRlkmGl/sk2uXmqn/6tJ3QLdAdArMt+a9kc=',
+              'size': 8134
+            }, {
+              'path': '2019-09-07-08-51-16-bridgedb-metrics',
+              'last_published': '2019-09-07 08:51',
+              'first_published': '2019-09-07 08:51',
+              'last_modified': '2019-11-28 19:40',
+              'sha256': 'bMceCkg/6qFiRL7TKBsfWlQs1MoYVngavRFSDcxa8Gc=',
+              'size': 7772
+            }
+          ]
+        }, {
+          'path': 'exit-lists',
+          'files': [
+            {
+              'path': '2019-11-27-23-02-00',
+              'last_published': '2019-11-27 23:02',
+              'first_published': '2019-11-27 23:02',
+              'last_modified': '2019-11-27 23:02',
+              'sha256': 'aOe9LSWJJ3N8sbembPFH8b027FlK2aixvKOkaDSEyUA=',
+              'types': ['tordnsel 1.0'],
+              'size': 160234
+            }, {
+              'path': '2019-11-28-00-02-00',
+              'last_published': '2019-11-28 00:02',
+              'first_published': '2019-11-28 00:02',
+              'last_modified': '2019-11-28 00:02',
+              'sha256': 'VuaqrB4wJid7RqHXn+BrL65F+FyKgMzzaj3XNumyOhc=',
+              'types': ['tordnsel 1.0'],
+              'size': 160234
             }
           ]
         }, {
@@ -367,51 +667,87 @@ EXAMPLE_INDEX = {
               'path': 'bandwidths',
               'files': [
                 {
-                  'path': '2019-07-03-00-30-57-bandwidth-616DF2278EE2C90F475E4EA2562E2C00EB9F10E517FB901229F331AEB70B8AD7',
-                  'size': 2580044,
-                  'last_modified': '2019-07-03 02:35'
-                }, {
-                  'path': '2019-07-03-01-35-02-bandwidth-2A5B679E9AA2D5C903CD16C00E16FEFA3E21E38AB3DE338941AE4AC13FD505DE',
-                  'size': 4210146,
-                  'last_modified': '2019-07-03 02:35'
-                }, {
-                  'path': '2019-07-03-01-35-03-bandwidth-3B06DAD9EDE8E1D08CC494E856AFD9D49256ECC2D68456799DD1EC7ED3436875',
-                  'size': 4249699,
-                  'last_modified': '2019-07-03 02:35'
+                  'path': '2019-11-27-20-44-46-bandwidth-572936DECB11EE615C0F698D0BB0E43D12F7A8B373F26236DBC4091099D3284A',
+                  'last_published': '2019-11-27 20:44',
+                  'first_published': '2019-11-27 20:44',
+                  'last_modified': '2019-11-27 22:35',
+                  'sha256': 'Ystd7EP1ElJ/SSLUAMklGtbhvgO2Qu2+9SIemBLieO8=',
+                  'types': ['bandwidth-file 1.0'],
+                  'size': 2510298
+                }, {
+                  'path': '2019-11-27-21-31-40-bandwidth-3A522F869B89A2DDB0A798C3878483A0A3CC1E8F02FDF1C81404A5F742729C29',
+                  'last_published': '2019-11-27 21:31',
+                  'first_published': '2019-11-27 21:31',
+                  'last_modified': '2019-11-27 22:35',
+                  'sha256': 'Wu1Fzp0qSXlNavxBAbm3TRj7AE5jGEdwrAYdoAhDMRE=',
+                  'types': ['bandwidth-file 1.0'],
+                  'size': 2450793
+                }, {
+                  'path': '2019-11-27-21-32-55-bandwidth-97EC4B3E17125BD9B58CE219FC417BA7B41C84D765711C8841AC75EB08CC5D8F',
+                  'last_published': '2019-11-27 21:32',
+                  'first_published': '2019-11-27 21:32',
+                  'last_modified': '2019-11-27 23:35',
+                  'sha256': '8VN/DJqQ0rmhfJvcr+yuwGSbvMDlkRfHyCnS5Y8GUuo=',
+                  'types': ['bandwidth-file 1.0'],
+                  'size': 2510643
                 }
               ]
             }, {
               'path': 'consensuses',
               'files': [
                 {
-                  'path': '2019-07-03-02-00-00-consensus',
-                  'size': 2211265,
-                  'last_modified': '2019-07-03 02:05'
-                }, {
-                  'path': '2019-07-03-03-00-00-consensus',
-                  'size': 2204478,
-                  'last_modified': '2019-07-03 03:05'
-                }, {
-                  'path': '2019-07-03-04-00-00-consensus',
-                  'size': 2202554,
-                  'last_modified': '2019-07-03 04:05'
+                  'path': '2019-11-27-23-00-00-consensus',
+                  'last_published': '2019-11-27 23:00',
+                  'first_published': '2019-11-27 23:00',
+                  'last_modified': '2019-11-27 23:05',
+                  'sha256': 'cGWT19Y0UVE/EUi3ZayacGvJU5t9T6MKaTOrNarAqlI=',
+                  'types': ['network-status-consensus-3 1.0'],
+                  'size': 2208505
+                }, {
+                  'path': '2019-11-28-00-00-00-consensus',
+                  'last_published': '2019-11-28 00:00',
+                  'first_published': '2019-11-28 00:00',
+                  'last_modified': '2019-11-28 00:05',
+                  'sha256': 'VHwgrdBK3QpD7NQypBPqeFPXUjfFnuSYtmp/36jBUOo=',
+                  'types': ['network-status-consensus-3 1.0'],
+                  'size': 2198681
+                }, {
+                  'path': '2019-11-28-01-00-00-consensus',
+                  'last_published': '2019-11-28 01:00',
+                  'first_published': '2019-11-28 01:00',
+                  'last_modified': '2019-11-28 01:05',
+                  'sha256': 'cXrIUvqiIybA87NoejHaBlDcN3VsLljGlmt+ku+HU/g=',
+                  'types': ['network-status-consensus-3 1.0'],
+                  'size': 2195362
                 }
               ]
             }, {
               'path': 'extra-infos',
               'files': [
                 {
-                  'path': '2019-07-03-02-05-00-extra-infos',
-                  'size': 1162899,
-                  'last_modified': '2019-07-03 02:05'
-                }, {
-                  'path': '2019-07-03-03-05-00-extra-infos',
-                  'size': 1133425,
-                  'last_modified': '2019-07-03 03:05'
-                }, {
-                  'path': '2019-07-03-04-05-00-extra-infos',
-                  'size': 1153793,
-                  'last_modified': '2019-07-03 04:05'
+                  'path': '2019-11-27-23-05-00-extra-infos',
+                  'last_published': '2019-11-27 23:28',
+                  'first_published': '2019-11-27 21:00',
+                  'last_modified': '2019-11-27 23:05',
+                  'sha256': 'xNzjDxg6iCQFdfShUD4e2oO+SiLxSgVqQp13dXDjwTU=',
+                  'types': ['extra-info 1.0'],
+                  'size': 1567496
+                }, {
+                  'path': '2019-11-28-00-05-00-extra-infos',
+                  'last_published': '2019-11-28 00:01',
+                  'first_published': '2019-11-27 19:04',
+                  'last_modified': '2019-11-28 00:05',
+                  'sha256': 'YybmmLOJgLf/kUyZ/IXQc5A4m7O8Sx0g6Ccu/G9ADhA=',
+                  'types': ['extra-info 1.0'],
+                  'size': 1471455
+                }, {
+                  'path': '2019-11-28-01-05-00-extra-infos',
+                  'last_published': '2019-11-28 01:17',
+                  'first_published': '2019-11-27 10:31',
+                  'last_modified': '2019-11-28 01:05',
+                  'sha256': '2nRmT48/8xVctXlhEHgawQVQy4DpcHS2NJHpQ1mZJRs=',
+                  'types': ['extra-info 1.0'],
+                  'size': 1378360
                 }
               ]
             }, {
@@ -421,34 +757,52 @@ EXAMPLE_INDEX = {
                   'path': 'consensus-microdesc',
                   'files': [
                     {
-                      'path': '2019-07-03-02-00-00-consensus-microdesc',
-                      'size': 2028994,
-                      'last_modified': '2019-07-03 02:05'
+                      'path': '2019-11-27-23-00-00-consensus-microdesc',
+                      'last_published': '2019-11-27 23:00',
+                      'first_published': '2019-11-27 23:00',
+                      'last_modified': '2019-11-27 23:05',
+                      'sha256': 'AEiZyuLlIf17aqIp3EOder5EzbSjgWesKaPLR/zRw6Q=',
+                      'types': ['network-status-microdesc-consensus-3 1.0'],
+                      'size': 2036635
                     }, {
-                      'path': '2019-07-03-03-00-00-consensus-microdesc',
-                      'size': 2022808,
-                      'last_modified': '2019-07-03 03:05'
+                      'path': '2019-11-28-00-00-00-consensus-microdesc',
+                      'last_published': '2019-11-28 00:00',
+                      'first_published': '2019-11-28 00:00',
+                      'last_modified': '2019-11-28 00:05',
+                      'sha256': '5P5AYHwuULmjNjdmubf0O6RYkIy7QAHLaIfv1GAUiYY=',
+                      'types': ['network-status-microdesc-consensus-3 1.0'],
+                      'size': 2029885
                     }, {
-                      'path': '2019-07-03-04-00-00-consensus-microdesc',
-                      'size': 2020751,
-                      'last_modified': '2019-07-03 04:05'
+                      'path': '2019-11-28-01-00-00-consensus-microdesc',
+                      'last_published': '2019-11-28 01:00',
+                      'first_published': '2019-11-28 01:00',
+                      'last_modified': '2019-11-28 01:05',
+                      'sha256': 'jjqsrEr3kkT7P04BlZmWICrmUxMkUp+vmtNQl93OnJc=',
+                      'types': ['network-status-microdesc-consensus-3 1.0'],
+                      'size': 2026257
                     }
                   ]
                 }, {
                   'path': 'micro',
                   'files': [
                     {
-                      'path': '2019-07-02-23-05-00-micro',
-                      'size': 19934,
-                      'last_modified': '2019-07-02 23:05'
+                      'path': '2019-11-27-22-05-00-micro',
+                      'last_modified': '2019-11-27 22:05',
+                      'sha256': 'cMrDxiUTrotiz80JaPQXEJ7UmNautHI+1tV7Jlvwn6Y=',
+                      'types': ['microdescriptor 1.0'],
+                      'size': 14940
                     }, {
-                      'path': '2019-07-03-00-05-00-micro',
-                      'size': 12030,
-                      'last_modified': '2019-07-03 00:05'
+                      'path': '2019-11-27-23-05-00-micro',
+                      'last_modified': '2019-11-27 23:05',
+                      'sha256': 'qzQ5EdHwdDNFAx6Ca7DgKdaABVznjmMMl+TNUMsKR1s=',
+                      'types': ['microdescriptor 1.0'],
+                      'size': 12234
                     }, {
-                      'path': '2019-07-03-01-05-00-micro',
-                      'size': 14850,
-                      'last_modified': '2019-07-03 01:05'
+                      'path': '2019-11-28-00-05-00-micro',
+                      'last_modified': '2019-11-28 00:05',
+                      'sha256': '9mnZR7wYjdQm259cay9YgGzxoGSqdjn1sl3oODEggDA=',
+                      'types': ['microdescriptor 1.0'],
+                      'size': 14696
                     }
                   ]
                 }
@@ -457,70 +811,144 @@ EXAMPLE_INDEX = {
               'path': 'server-descriptors',
               'files': [
                 {
-                  'path': '2019-07-03-02-05-00-server-descriptors',
-                  'size': 1374587,
-                  'last_modified': '2019-07-03 02:05'
-                }, {
-                  'path': '2019-07-03-03-05-00-server-descriptors',
-                  'size': 1353286,
-                  'last_modified': '2019-07-03 03:05'
-                }, {
-                  'path': '2019-07-03-04-05-00-server-descriptors',
-                  'size': 1336125,
-                  'last_modified': '2019-07-03 04:05'
+                  'path': '2019-11-27-23-05-00-server-descriptors',
+                  'last_published': '2019-11-27 23:28',
+                  'first_published': '2019-11-27 21:00',
+                  'last_modified': '2019-11-27 23:05',
+                  'sha256': 'HLb5s027ceH0iNag9IzQxR42pSMWreTxtC8xuNyNMs0=',
+                  'types': ['server-descriptor 1.0'],
+                  'size': 1978883
+                }, {
+                  'path': '2019-11-28-00-05-00-server-descriptors',
+                  'last_published': '2019-11-28 00:01',
+                  'first_published': '2019-11-27 19:04',
+                  'last_modified': '2019-11-28 00:05',
+                  'sha256': '35HKfcpwZpOP6DU/6078dciDIfyt/7rbVfL6qOPphrM=',
+                  'types': ['server-descriptor 1.0'],
+                  'size': 1780252
+                }, {
+                  'path': '2019-11-28-01-05-00-server-descriptors',
+                  'last_published': '2019-11-28 01:17',
+                  'first_published': '2019-11-27 10:31',
+                  'last_modified': '2019-11-28 01:05',
+                  'sha256': 'gzhTR8EVKPnki8xT0XHcw2WV+BVmWJ6RxQjjhyxZ8Ro=',
+                  'types': ['server-descriptor 1.0'],
+                  'size': 1737656
                 }
               ]
             }, {
               'path': 'votes',
               'files': [
                 {
-                  'path': '2019-07-03-02-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-D9B6923A3AF9FB5DB3CD30191BCBD29A7A936D86',
-                  'size': 3449707,
-                  'last_modified': '2019-07-03 02:05'
-                }, {
-                  'path': '2019-07-03-02-00-00-vote-14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4-39429F0789A9EC07A2A8754C4C449CCABAB787CC',
-                  'size': 3435482,
-                  'last_modified': '2019-07-03 02:05'
-                }, {
-                  'path': '2019-07-03-02-00-00-vote-23D15D965BC35114467363C165C4F724B64B4F66-4337842B697467123B5351E1E8EB64ED497C0EDA',
-                  'size': 3571106,
-                  'last_modified': '2019-07-03 02:05'
+                  'path': '2019-11-27-23-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-82EBEC114F0D555B5A44C13533235AFC68BE707D',
+                  'last_published': '2019-11-27 23:00',
+                  'first_published': '2019-11-27 23:00',
+                  'last_modified': '2019-11-27 23:05',
+                  'sha256': 'uBb0SCyaR+hRrJBoHpcgzDtdKzZD7bsqWVAf4ivYNao=',
+                  'types': ['network-status-vote-3 1.0'],
+                  'size': 3721766
+                }, {
+                  'path': '2019-11-27-23-00-00-vote-14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4-1E73049822816A032C719782F7EBA461E1A43055',
+                  'last_published': '2019-11-27 23:00',
+                  'first_published': '2019-11-27 23:00',
+                  'last_modified': '2019-11-27 23:05',
+                  'sha256': 'WyUizNcJZIvKfJTc9cwUy1iyvSxfU5gAZGf3dSVQrMA=',
+                  'types': ['network-status-vote-3 1.0'],
+                  'size': 3396129
+                }, {
+                  'path': '2019-11-27-23-00-00-vote-23D15D965BC35114467363C165C4F724B64B4F66-7D0503631CE3246ABD7A52BFAA4B416F02DDA5FA',
+                  'last_published': '2019-11-27 23:00',
+                  'first_published': '2019-11-27 23:00',
+                  'last_modified': '2019-11-27 23:05',
+                  'sha256': 'CIYCy5yb3FtJSpt8oDcqHUkuSnugtn6TnOvvR5XxGsk=',
+                  'types': ['network-status-vote-3 1.0'],
+                  'size': 3432263
                 }
               ]
             }
           ]
         }, {
+          'path': 'snowflakes',
+          'files': [
+            {
+              'path': '2019-11-28-08-24-18-snowflake-stats',
+              'last_published': '2019-11-28 08:24',
+              'first_published': '2019-11-28 08:24',
+              'last_modified': '2019-11-28 08:40',
+              'sha256': 'ikyXH8BpDQ03oZUw0P1+WyTLzoqQUma4mQ3ULJl74L4=',
+              'types': ['snowflake-stats 1.0'],
+              'size': 750
+            }, {
+              'path': '2019-11-29-08-24-18-snowflake-stats',
+              'last_published': '2019-11-29 08:24',
+              'first_published': '2019-11-29 08:24',
+              'last_modified': '2019-11-29 08:40',
+              'sha256': 'J5XTwjE1t/4mc7XLzr1mmYtoizIPH0WUdwNZs041nE0=',
+              'types': ['snowflake-stats 1.0'],
+              'size': 725
+            }, {
+              'path': '2019-11-30-08-24-18-snowflake-stats',
+              'last_published': '2019-11-30 08:24',
+              'first_published': '2019-11-30 08:24',
+              'last_modified': '2019-11-30 08:40',
+              'sha256': 'LyIeFYoZ+qnsXXZkLWkqXpqZ81eXorecQA6JOpi6kbw=',
+              'types': ['snowflake-stats 1.0'],
+              'size': 734
+            }
+          ]
+        }, {
           'path': 'torperf',
           'files': [
             {
-              'path': 'op-ab-1048576-2019-07-03.tpf',
-              'size': 39152,
-              'last_modified': '2019-07-04 00:01'
+              'path': 'op-ab-1048576-2019-11-27.tpf',
+              'last_published': '2019-11-27 23:59',
+              'first_published': '2019-11-27 00:19',
+              'last_modified': '2019-11-28 00:01',
+              'sha256': 'EIUcAWfl8hXt5Ho4r7UpNptBwJKC5VW+xtXDcsxIftI=',
+              'types': ['torperf 1.1'],
+              'size': 42012
             }, {
-              'path': 'op-ab-51200-2019-07-03.tpf',
-              'size': 248173,
-              'last_modified': '2019-07-04 00:01'
+              'path': 'op-ab-1048576-2019-11-28.tpf',
+              'last_published': '2019-11-28 23:59',
+              'first_published': '2019-11-28 00:24',
+              'last_modified': '2019-11-29 00:01',
+              'sha256': '88mSSG4Vjl6Ue6TQ3ZggEmUE5N0/OZVnApJjhReegPg=',
+              'types': ['torperf 1.1'],
+              'size': 39910
             }, {
-              'path': 'op-ab-5242880-2019-07-03.tpf',
-              'size': 15900,
-              'last_modified': '2019-07-04 00:01'
+              'path': 'op-ab-1048576-2019-11-29.tpf',
+              'last_published': '2019-11-29 22:44',
+              'first_published': '2019-11-29 00:04',
+              'last_modified': '2019-11-30 00:01',
+              'sha256': 'AdpYLfjx8dbyUgxg/wr/FJh0e/wNAuhffKp9rwdAzuk=',
+              'types': ['torperf 1.1'],
+              'size': 47405
             }
           ]
         }, {
           'path': 'webstats',
           'files': [
             {
-              'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190620.xz',
-              'size': 8872,
-              'last_modified': '2019-07-03 04:21'
+              'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20191116.xz',
+              'last_published': '2019-11-16 00:00',
+              'first_published': '2019-11-16 00:00',
+              'last_modified': '2019-11-28 19:41',
+              'sha256': 'mJdqQBAfEPze/TuhLxBy9//1CFwCvbxHiIwwqptt1ek=',
+              'size': 10480
             }, {
-              'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190621.xz',
-              'size': 9312,
-              'last_modified': '2019-07-04 04:21'
+              'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20191117.xz',
+              'last_published': '2019-11-17 00:00',
+              'first_published': '2019-11-17 00:00',
+              'last_modified': '2019-11-30 19:41',
+              'sha256': '0vAFfbUWeoDn2kZyxK1a9CDZwTk5/yc9yz9IjEcPolo=',
+              'size': 11836
             }, {
-              'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20190622.xz',
-              'size': 9836,
-              'last_modified': '2019-07-05 04:21'
+              'path': '2019.www.torproject.org_hetzner-hel1-03.torproject.org_access.log_20191118.xz',
+              'last_published': '2019-11-18 00:00',
+              'first_published': '2019-11-18 00:00',
+              'last_modified': '2019-11-30 19:41',
+              'sha256': 'CcIuv30SczF0l6Z41d90+ykLobsRuqTfNT9TrfimcKk=',
+              'size': 11108
             }
           ]
         }

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits