[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [tor/master] Merge branch 'networkstatus_mmap' into networkstatus_mmap_merge



commit 988d4903a3fc23153896e0daf7738f87ade9bc4b
Merge: a18230115 594140574
Author: Nick Mathewson <nickm@xxxxxxxxxxxxxx>
Date:   Wed Oct 31 09:04:12 2018 -0400

    Merge branch 'networkstatus_mmap' into networkstatus_mmap_merge

 changes/feature27244                  |  5 ++
 src/feature/control/control.c         |  6 ++-
 src/feature/dirauth/dirvote.c         | 19 ++++---
 src/feature/dircache/consdiffmgr.c    | 83 +++++++++++++++++++++---------
 src/feature/dircache/consdiffmgr.h    | 11 +++-
 src/feature/dircache/dirserv.c        |  5 +-
 src/feature/dircache/dirserv.h        |  1 +
 src/feature/dirclient/dirclient.c     | 22 +++++---
 src/feature/dircommon/consdiff.c      | 42 ++++++++-------
 src/feature/dircommon/consdiff.h      | 15 +++---
 src/feature/dirparse/authcert_parse.c | 16 +++---
 src/feature/dirparse/authcert_parse.h |  1 +
 src/feature/dirparse/ns_parse.c       | 55 ++++++++++++--------
 src/feature/dirparse/ns_parse.h       | 10 ++--
 src/feature/nodelist/authcert.c       |  3 +-
 src/feature/nodelist/networkstatus.c  | 96 +++++++++++++++++++----------------
 src/feature/nodelist/networkstatus.h  |  4 +-
 src/feature/relay/router.c            |  2 +-
 src/test/bench.c                      |  6 ++-
 src/test/fuzz/fuzz_consensus.c        |  6 +--
 src/test/fuzz/fuzz_diff.c             | 32 +++++++-----
 src/test/fuzz/fuzz_diff_apply.c       | 13 +++--
 src/test/fuzz/fuzz_vrs.c              | 16 +++---
 src/test/test_consdiff.c              | 94 +++++++++++++++++++++++-----------
 src/test/test_consdiffmgr.c           | 41 +++++++++++----
 src/test/test_dir.c                   | 44 ++++++++++++----
 src/test/test_dir_common.c            | 17 +++++--
 src/test/test_dir_handle_get.c        | 18 +++++--
 src/test/test_routerlist.c            | 19 +++++--
 src/test/test_shared_random.c         | 12 +++--
 30 files changed, 467 insertions(+), 247 deletions(-)

diff --cc src/feature/control/control.c
index f0db97dc8,9e7d21308..3fa47747e
--- a/src/feature/control/control.c
+++ b/src/feature/control/control.c
@@@ -2352,7 -2341,9 +2352,11 @@@ getinfo_helper_dir(control_connection_
          *answer = tor_strdup(consensus->dir);
      }
      if (!*answer) { /* try loading it from disk */
-       *answer = networkstatus_read_cached_consensus("ns");
 -      char *filename = get_cachedir_fname("cached-consensus");
 -      *answer = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
 -      tor_free(filename);
++      tor_mmap_t *mapped = networkstatus_map_cached_consensus("ns");
++      if (mapped) {
++        *answer = tor_memdup_nulterm(mapped->data, mapped->size);
++        tor_munmap_file(mapped);
++      }
        if (!*answer) { /* generate an error */
          *errmsg = "Could not open cached consensus. "
            "Make sure FetchUselessDescriptors is set to 1.";
diff --cc src/feature/dircache/dirserv.c
index 57178cd50,433d3f4ce..4366000e2
--- a/src/feature/dircache/dirserv.c
+++ b/src/feature/dircache/dirserv.c
@@@ -95,173 -1133,1591 +95,176 @@@ directory_fetches_from_authorities(cons
    me = router_get_my_routerinfo();
    if (!me || (!me->supports_tunnelled_dir_requests && !refuseunknown))
      return 0; /* if we don't service directory requests, return 0 too */
 -  return 1;
 -}
 -
 -/** Return 1 if we should fetch new networkstatuses, descriptors, etc
 - * on the "mirror" schedule rather than the "client" schedule.
 - */
 -int
 -directory_fetches_dir_info_early(const or_options_t *options)
 -{
 -  return directory_fetches_from_authorities(options);
 -}
 -
 -/** Return 1 if we should fetch new networkstatuses, descriptors, etc
 - * on a very passive schedule -- waiting long enough for ordinary clients
 - * to probably have the info we want. These would include bridge users,
 - * and maybe others in the future e.g. if a Tor client uses another Tor
 - * client as a directory guard.
 - */
 -int
 -directory_fetches_dir_info_later(const or_options_t *options)
 -{
 -  return options->UseBridges != 0;
 -}
 -
 -/** Return true iff we want to serve certificates for authorities
 - * that we don't acknowledge as authorities ourself.
 - * Use we_want_to_fetch_unknown_auth_certs to check if we want to fetch
 - * and keep these certificates.
 - */
 -int
 -directory_caches_unknown_auth_certs(const or_options_t *options)
 -{
 -  return dir_server_mode(options) || options->BridgeRelay;
 -}
 -
 -/** Return 1 if we want to fetch and serve descriptors, networkstatuses, etc
 - * Else return 0.
 - * Check options->DirPort_set and directory_permits_begindir_requests()
 - * to see if we are willing to serve these directory documents to others via
 - * the DirPort and begindir-over-ORPort, respectively.
 - *
 - * To check if we should fetch documents, use we_want_to_fetch_flavor and
 - * we_want_to_fetch_unknown_auth_certs instead of this function.
 - */
 -int
 -directory_caches_dir_info(const or_options_t *options)
 -{
 -  if (options->BridgeRelay || dir_server_mode(options))
 -    return 1;
 -  if (!server_mode(options) || !advertised_server_mode())
 -    return 0;
 -  /* We need an up-to-date view of network info if we're going to try to
 -   * block exit attempts from unknown relays. */
 -  return ! router_my_exit_policy_is_reject_star() &&
 -    should_refuse_unknown_exits(options);
 -}
 -
 -/** Return 1 if we want to allow remote clients to ask us directory
 - * requests via the "begin_dir" interface, which doesn't require
 - * having any separate port open. */
 -int
 -directory_permits_begindir_requests(const or_options_t *options)
 -{
 -  return options->BridgeRelay != 0 || dir_server_mode(options);
 -}
 -
 -/** Return 1 if we have no need to fetch new descriptors. This generally
 - * happens when we're not a dir cache and we haven't built any circuits
 - * lately.
 - */
 -int
 -directory_too_idle_to_fetch_descriptors(const or_options_t *options,
 -                                        time_t now)
 -{
 -  return !directory_caches_dir_info(options) &&
 -         !options->FetchUselessDescriptors &&
 -         rep_hist_circbuilding_dormant(now);
 -}
 -
 -/********************************************************************/
 -
 -/** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
 - * currently serving. */
 -static strmap_t *cached_consensuses = NULL;
 -
 -/** Decrement the reference count on <b>d</b>, and free it if it no longer has
 - * any references. */
 -void
 -cached_dir_decref(cached_dir_t *d)
 -{
 -  if (!d || --d->refcnt > 0)
 -    return;
 -  clear_cached_dir(d);
 -  tor_free(d);
 -}
 -
 -/** Allocate and return a new cached_dir_t containing the string <b>s</b>,
 - * published at <b>published</b>. */
 -cached_dir_t *
 -new_cached_dir(char *s, time_t published)
 -{
 -  cached_dir_t *d = tor_malloc_zero(sizeof(cached_dir_t));
 -  d->refcnt = 1;
 -  d->dir = s;
 -  d->dir_len = strlen(s);
 -  d->published = published;
 -  if (tor_compress(&(d->dir_compressed), &(d->dir_compressed_len),
 -                   d->dir, d->dir_len, ZLIB_METHOD)) {
 -    log_warn(LD_BUG, "Error compressing directory");
 -  }
 -  return d;
 -}
 -
 -/** Remove all storage held in <b>d</b>, but do not free <b>d</b> itself. */
 -static void
 -clear_cached_dir(cached_dir_t *d)
 -{
 -  tor_free(d->dir);
 -  tor_free(d->dir_compressed);
 -  memset(d, 0, sizeof(cached_dir_t));
 -}
 -
 -/** Free all storage held by the cached_dir_t in <b>d</b>. */
 -static void
 -free_cached_dir_(void *_d)
 -{
 -  cached_dir_t *d;
 -  if (!_d)
 -    return;
 -
 -  d = (cached_dir_t *)_d;
 -  cached_dir_decref(d);
 -}
 -
 -/** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that
 - * we're serving with <b>networkstatus</b>, published at <b>published</b>.  No
 - * validation is performed. */
 -void
 -dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
 -                                           size_t networkstatus_len,
 -                                           const char *flavor_name,
 -                                           const common_digests_t *digests,
 -                                           const uint8_t *sha3_as_signed,
 -                                           time_t published)
 -{
 -  cached_dir_t *new_networkstatus;
 -  cached_dir_t *old_networkstatus;
 -  if (!cached_consensuses)
 -    cached_consensuses = strmap_new();
 -
 -  new_networkstatus =
 -    new_cached_dir(tor_memdup_nulterm(networkstatus, networkstatus_len),
 -                   published);
 -  memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t));
 -  memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed,
 -         DIGEST256_LEN);
 -  old_networkstatus = strmap_set(cached_consensuses, flavor_name,
 -                                 new_networkstatus);
 -  if (old_networkstatus)
 -    cached_dir_decref(old_networkstatus);
 -}
 -
 -/** Return the latest downloaded consensus networkstatus in encoded, signed,
 - * optionally compressed format, suitable for sending to clients. */
 -cached_dir_t *
 -dirserv_get_consensus(const char *flavor_name)
 -{
 -  if (!cached_consensuses)
 -    return NULL;
 -  return strmap_get(cached_consensuses, flavor_name);
 -}
 -
 -/** If a router's uptime is at least this value, then it is always
 - * considered stable, regardless of the rest of the network. This
 - * way we resist attacks where an attacker doubles the size of the
 - * network using allegedly high-uptime nodes, displacing all the
 - * current guards. */
 -#define UPTIME_TO_GUARANTEE_STABLE (3600*24*30)
 -/** If a router's MTBF is at least this value, then it is always stable.
 - * See above.  (Corresponds to about 7 days for current decay rates.) */
 -#define MTBF_TO_GUARANTEE_STABLE (60*60*24*5)
 -/** Similarly, every node with at least this much weighted time known can be
 - * considered familiar enough to be a guard.  Corresponds to about 20 days for
 - * current decay rates.
 - */
 -#define TIME_KNOWN_TO_GUARANTEE_FAMILIAR (8*24*60*60)
 -/** Similarly, every node with sufficient WFU is around enough to be a guard.
 - */
 -#define WFU_TO_GUARANTEE_GUARD (0.98)
 -
 -/* Thresholds for server performance: set by
 - * dirserv_compute_performance_thresholds, and used by
 - * generate_v2_networkstatus */
 -
 -/** Any router with an uptime of at least this value is stable. */
 -static uint32_t stable_uptime = 0; /* start at a safe value */
 -/** Any router with an mtbf of at least this value is stable. */
 -static double stable_mtbf = 0.0;
 -/** If true, we have measured enough mtbf info to look at stable_mtbf rather
 - * than stable_uptime. */
 -static int enough_mtbf_info = 0;
 -/** Any router with a weighted fractional uptime of at least this much might
 - * be good as a guard. */
 -static double guard_wfu = 0.0;
 -/** Don't call a router a guard unless we've known about it for at least this
 - * many seconds. */
 -static long guard_tk = 0;
 -/** Any router with a bandwidth at least this high is "Fast" */
 -static uint32_t fast_bandwidth_kb = 0;
 -/** If exits can be guards, then all guards must have a bandwidth this
 - * high. */
 -static uint32_t guard_bandwidth_including_exits_kb = 0;
 -/** If exits can't be guards, then all guards must have a bandwidth this
 - * high. */
 -static uint32_t guard_bandwidth_excluding_exits_kb = 0;
 -
 -/** Helper: estimate the uptime of a router given its stated uptime and the
 - * amount of time since it last stated its stated uptime. */
 -static inline long
 -real_uptime(const routerinfo_t *router, time_t now)
 -{
 -  if (now < router->cache_info.published_on)
 -    return router->uptime;
 -  else
 -    return router->uptime + (now - router->cache_info.published_on);
 -}
 -
 -/** Return 1 if <b>router</b> is not suitable for these parameters, else 0.
 - * If <b>need_uptime</b> is non-zero, we require a minimum uptime.
 - * If <b>need_capacity</b> is non-zero, we require a minimum advertised
 - * bandwidth.
 - */
 -static int
 -dirserv_thinks_router_is_unreliable(time_t now,
 -                                    routerinfo_t *router,
 -                                    int need_uptime, int need_capacity)
 -{
 -  if (need_uptime) {
 -    if (!enough_mtbf_info) {
 -      /* XXXX We should change the rule from
 -       * "use uptime if we don't have mtbf data" to "don't advertise Stable on
 -       * v3 if we don't have enough mtbf data."  Or maybe not, since if we ever
 -       * hit a point where we need to reset a lot of authorities at once,
 -       * none of them would be in a position to declare Stable.
 -       */
 -      long uptime = real_uptime(router, now);
 -      if ((unsigned)uptime < stable_uptime &&
 -          (unsigned)uptime < UPTIME_TO_GUARANTEE_STABLE)
 -        return 1;
 -    } else {
 -      double mtbf =
 -        rep_hist_get_stability(router->cache_info.identity_digest, now);
 -      if (mtbf < stable_mtbf &&
 -          mtbf < MTBF_TO_GUARANTEE_STABLE)
 -        return 1;
 -    }
 -  }
 -  if (need_capacity) {
 -    uint32_t bw_kb = dirserv_get_credible_bandwidth_kb(router);
 -    if (bw_kb < fast_bandwidth_kb)
 -      return 1;
 -  }
 -  return 0;
 -}
 -
 -/** Return true iff <b>router</b> should be assigned the "HSDir" flag.
 - *
 - * Right now this means it advertises support for it, it has a high uptime,
 - * it's a directory cache, it has the Stable and Fast flags, and it's currently
 - * considered Running.
 - *
 - * This function needs to be called after router-\>is_running has
 - * been set.
 - */
 -static int
 -dirserv_thinks_router_is_hs_dir(const routerinfo_t *router,
 -                                const node_t *node, time_t now)
 -{
 -
 -  long uptime;
 -
 -  /* If we haven't been running for at least
 -   * get_options()->MinUptimeHidServDirectoryV2 seconds, we can't
 -   * have accurate data telling us a relay has been up for at least
 -   * that long. We also want to allow a bit of slack: Reachability
 -   * tests aren't instant. If we haven't been running long enough,
 -   * trust the relay. */
 -
 -  if (get_uptime() >
 -      get_options()->MinUptimeHidServDirectoryV2 * 1.1)
 -    uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now),
 -                 real_uptime(router, now));
 -  else
 -    uptime = real_uptime(router, now);
 -
 -  return (router->wants_to_be_hs_dir &&
 -          router->supports_tunnelled_dir_requests &&
 -          node->is_stable && node->is_fast &&
 -          uptime >= get_options()->MinUptimeHidServDirectoryV2 &&
 -          router_is_active(router, node, now));
 -}
 -
 -/** Don't consider routers with less bandwidth than this when computing
 - * thresholds. */
 -#define ABSOLUTE_MIN_BW_VALUE_TO_CONSIDER_KB 4
 -
 -/** Helper for dirserv_compute_performance_thresholds(): Decide whether to
 - * include a router in our calculations, and return true iff we should; the
 - * require_mbw parameter is passed in by
 - * dirserv_compute_performance_thresholds() and controls whether we ever
 - * count routers with only advertised bandwidths */
 -static int
 -router_counts_toward_thresholds(const node_t *node, time_t now,
 -                                const digestmap_t *omit_as_sybil,
 -                                int require_mbw)
 -{
 -  /* Have measured bw? */
 -  int have_mbw =
 -    dirserv_has_measured_bw(node->identity);
 -  uint64_t min_bw_kb = ABSOLUTE_MIN_BW_VALUE_TO_CONSIDER_KB;
 -  const or_options_t *options = get_options();
 -
 -  if (options->TestingTorNetwork) {
 -    min_bw_kb = (int64_t)options->TestingMinExitFlagThreshold / 1000;
 -  }
 -
 -  return node->ri && router_is_active(node->ri, node, now) &&
 -    !digestmap_get(omit_as_sybil, node->identity) &&
 -    (dirserv_get_credible_bandwidth_kb(node->ri) >= min_bw_kb) &&
 -    (have_mbw || !require_mbw);
 -}
 -
 -/** Look through the routerlist, and using the measured bandwidth cache count
 - * how many measured bandwidths we know.  This is used to decide whether we
 - * ever trust advertised bandwidths for purposes of assigning flags. */
 -void
 -dirserv_count_measured_bws(const smartlist_t *routers)
 -{
 -  /* Initialize this first */
 -  routers_with_measured_bw = 0;
 -
 -  /* Iterate over the routerlist and count measured bandwidths */
 -  SMARTLIST_FOREACH_BEGIN(routers, const routerinfo_t *, ri) {
 -    /* Check if we know a measured bandwidth for this one */
 -    if (dirserv_has_measured_bw(ri->cache_info.identity_digest)) {
 -      ++routers_with_measured_bw;
 -    }
 -  } SMARTLIST_FOREACH_END(ri);
 -}
 -
 -/** Look through the routerlist, the Mean Time Between Failure history, and
 - * the Weighted Fractional Uptime history, and use them to set thresholds for
 - * the Stable, Fast, and Guard flags.  Update the fields stable_uptime,
 - * stable_mtbf, enough_mtbf_info, guard_wfu, guard_tk, fast_bandwidth,
 - * guard_bandwidth_including_exits, and guard_bandwidth_excluding_exits.
 - *
 - * Also, set the is_exit flag of each router appropriately. */
 -void
 -dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
 -{
 -  int n_active, n_active_nonexit, n_familiar;
 -  uint32_t *uptimes, *bandwidths_kb, *bandwidths_excluding_exits_kb;
 -  long *tks;
 -  double *mtbfs, *wfus;
 -  smartlist_t *nodelist;
 -  time_t now = time(NULL);
 -  const or_options_t *options = get_options();
 -
 -  /* Require mbw? */
 -  int require_mbw =
 -    (routers_with_measured_bw >
 -     options->MinMeasuredBWsForAuthToIgnoreAdvertised) ? 1 : 0;
 -
 -  /* initialize these all here, in case there are no routers */
 -  stable_uptime = 0;
 -  stable_mtbf = 0;
 -  fast_bandwidth_kb = 0;
 -  guard_bandwidth_including_exits_kb = 0;
 -  guard_bandwidth_excluding_exits_kb = 0;
 -  guard_tk = 0;
 -  guard_wfu = 0;
 -
 -  nodelist_assert_ok();
 -  nodelist = nodelist_get_list();
 -
 -  /* Initialize arrays that will hold values for each router.  We'll
 -   * sort them and use that to compute thresholds. */
 -  n_active = n_active_nonexit = 0;
 -  /* Uptime for every active router. */
 -  uptimes = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
 -  /* Bandwidth for every active router. */
 -  bandwidths_kb = tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
 -  /* Bandwidth for every active non-exit router. */
 -  bandwidths_excluding_exits_kb =
 -    tor_calloc(smartlist_len(nodelist), sizeof(uint32_t));
 -  /* Weighted mean time between failure for each active router. */
 -  mtbfs = tor_calloc(smartlist_len(nodelist), sizeof(double));
 -  /* Time-known for each active router. */
 -  tks = tor_calloc(smartlist_len(nodelist), sizeof(long));
 -  /* Weighted fractional uptime for each active router. */
 -  wfus = tor_calloc(smartlist_len(nodelist), sizeof(double));
 -
 -  /* Now, fill in the arrays. */
 -  SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
 -    if (options->BridgeAuthoritativeDir &&
 -        node->ri &&
 -        node->ri->purpose != ROUTER_PURPOSE_BRIDGE)
 -      continue;
 -
 -    routerinfo_t *ri = node->ri;
 -    if (ri) {
 -      node->is_exit = (!router_exit_policy_rejects_all(ri) &&
 -                       exit_policy_is_general_exit(ri->exit_policy));
 -    }
 -
 -    if (router_counts_toward_thresholds(node, now, omit_as_sybil,
 -                                        require_mbw)) {
 -      const char *id = node->identity;
 -      uint32_t bw_kb;
 -
 -      /* resolve spurious clang shallow analysis null pointer errors */
 -      tor_assert(ri);
 -
 -      uptimes[n_active] = (uint32_t)real_uptime(ri, now);
 -      mtbfs[n_active] = rep_hist_get_stability(id, now);
 -      tks  [n_active] = rep_hist_get_weighted_time_known(id, now);
 -      bandwidths_kb[n_active] = bw_kb = dirserv_get_credible_bandwidth_kb(ri);
 -      if (!node->is_exit || node->is_bad_exit) {
 -        bandwidths_excluding_exits_kb[n_active_nonexit] = bw_kb;
 -        ++n_active_nonexit;
 -      }
 -      ++n_active;
 -    }
 -  } SMARTLIST_FOREACH_END(node);
 -
 -  /* Now, compute thresholds. */
 -  if (n_active) {
 -    /* The median uptime is stable. */
 -    stable_uptime = median_uint32(uptimes, n_active);
 -    /* The median mtbf is stable, if we have enough mtbf info */
 -    stable_mtbf = median_double(mtbfs, n_active);
 -    /* The 12.5th percentile bandwidth is fast. */
 -    fast_bandwidth_kb = find_nth_uint32(bandwidths_kb, n_active, n_active/8);
 -    /* (Now bandwidths is sorted.) */
 -    if (fast_bandwidth_kb < RELAY_REQUIRED_MIN_BANDWIDTH/(2 * 1000))
 -      fast_bandwidth_kb = bandwidths_kb[n_active/4];
 -    guard_bandwidth_including_exits_kb =
 -      third_quartile_uint32(bandwidths_kb, n_active);
 -    guard_tk = find_nth_long(tks, n_active, n_active/8);
 -  }
 -
 -  if (guard_tk > TIME_KNOWN_TO_GUARANTEE_FAMILIAR)
 -    guard_tk = TIME_KNOWN_TO_GUARANTEE_FAMILIAR;
 -
 -  {
 -    /* We can vote on a parameter for the minimum and maximum. */
 -#define ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG 4
 -    int32_t min_fast_kb, max_fast_kb, min_fast, max_fast;
 -    min_fast = networkstatus_get_param(NULL, "FastFlagMinThreshold",
 -      ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG,
 -      ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG,
 -      INT32_MAX);
 -    if (options->TestingTorNetwork) {
 -      min_fast = (int32_t)options->TestingMinFastFlagThreshold;
 -    }
 -    max_fast = networkstatus_get_param(NULL, "FastFlagMaxThreshold",
 -                                       INT32_MAX, min_fast, INT32_MAX);
 -    min_fast_kb = min_fast / 1000;
 -    max_fast_kb = max_fast / 1000;
 -
 -    if (fast_bandwidth_kb < (uint32_t)min_fast_kb)
 -      fast_bandwidth_kb = min_fast_kb;
 -    if (fast_bandwidth_kb > (uint32_t)max_fast_kb)
 -      fast_bandwidth_kb = max_fast_kb;
 -  }
 -  /* Protect sufficiently fast nodes from being pushed out of the set
 -   * of Fast nodes. */
 -  if (options->AuthDirFastGuarantee &&
 -      fast_bandwidth_kb > options->AuthDirFastGuarantee/1000)
 -    fast_bandwidth_kb = (uint32_t)options->AuthDirFastGuarantee/1000;
 -
 -  /* Now that we have a time-known that 7/8 routers are known longer than,
 -   * fill wfus with the wfu of every such "familiar" router. */
 -  n_familiar = 0;
 -
 -  SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
 -      if (router_counts_toward_thresholds(node, now,
 -                                          omit_as_sybil, require_mbw)) {
 -        routerinfo_t *ri = node->ri;
 -        const char *id = ri->cache_info.identity_digest;
 -        long tk = rep_hist_get_weighted_time_known(id, now);
 -        if (tk < guard_tk)
 -          continue;
 -        wfus[n_familiar++] = rep_hist_get_weighted_fractional_uptime(id, now);
 -      }
 -  } SMARTLIST_FOREACH_END(node);
 -  if (n_familiar)
 -    guard_wfu = median_double(wfus, n_familiar);
 -  if (guard_wfu > WFU_TO_GUARANTEE_GUARD)
 -    guard_wfu = WFU_TO_GUARANTEE_GUARD;
 -
 -  enough_mtbf_info = rep_hist_have_measured_enough_stability();
 -
 -  if (n_active_nonexit) {
 -    guard_bandwidth_excluding_exits_kb =
 -      find_nth_uint32(bandwidths_excluding_exits_kb,
 -                      n_active_nonexit, n_active_nonexit*3/4);
 -  }
 -
 -  log_info(LD_DIRSERV,
 -      "Cutoffs: For Stable, %lu sec uptime, %lu sec MTBF. "
 -      "For Fast: %lu kilobytes/sec. "
 -      "For Guard: WFU %.03f%%, time-known %lu sec, "
 -      "and bandwidth %lu or %lu kilobytes/sec. "
 -      "We%s have enough stability data.",
 -      (unsigned long)stable_uptime,
 -      (unsigned long)stable_mtbf,
 -      (unsigned long)fast_bandwidth_kb,
 -      guard_wfu*100,
 -      (unsigned long)guard_tk,
 -      (unsigned long)guard_bandwidth_including_exits_kb,
 -      (unsigned long)guard_bandwidth_excluding_exits_kb,
 -      enough_mtbf_info ? "" : " don't");
 -
 -  tor_free(uptimes);
 -  tor_free(mtbfs);
 -  tor_free(bandwidths_kb);
 -  tor_free(bandwidths_excluding_exits_kb);
 -  tor_free(tks);
 -  tor_free(wfus);
 -}
 -
 -/* Use dirserv_compute_performance_thresholds() to compute the thresholds
 - * for the status flags, specifically for bridges.
 - *
 - * This is only called by a Bridge Authority from
 - * networkstatus_getinfo_by_purpose().
 - */
 -void
 -dirserv_compute_bridge_flag_thresholds(void)
 -{
 -  digestmap_t *omit_as_sybil = digestmap_new();
 -  dirserv_compute_performance_thresholds(omit_as_sybil);
 -  digestmap_free(omit_as_sybil, NULL);
 -}
 -
 -/** Measured bandwidth cache entry */
 -typedef struct mbw_cache_entry_s {
 -  long mbw_kb;
 -  time_t as_of;
 -} mbw_cache_entry_t;
 -
 -/** Measured bandwidth cache - keys are identity_digests, values are
 - * mbw_cache_entry_t *. */
 -static digestmap_t *mbw_cache = NULL;
 -
 -/** Store a measured bandwidth cache entry when reading the measured
 - * bandwidths file. */
 -STATIC void
 -dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
 -                          time_t as_of)
 -{
 -  mbw_cache_entry_t *e = NULL;
 -
 -  tor_assert(parsed_line);
 -
 -  /* Allocate a cache if we need */
 -  if (!mbw_cache) mbw_cache = digestmap_new();
 -
 -  /* Check if we have an existing entry */
 -  e = digestmap_get(mbw_cache, parsed_line->node_id);
 -  /* If we do, we can re-use it */
 -  if (e) {
 -    /* Check that we really are newer, and update */
 -    if (as_of > e->as_of) {
 -      e->mbw_kb = parsed_line->bw_kb;
 -      e->as_of = as_of;
 -    }
 -  } else {
 -    /* We'll have to insert a new entry */
 -    e = tor_malloc(sizeof(*e));
 -    e->mbw_kb = parsed_line->bw_kb;
 -    e->as_of = as_of;
 -    digestmap_set(mbw_cache, parsed_line->node_id, e);
 -  }
 -}
 -
 -/** Clear and free the measured bandwidth cache */
 -void
 -dirserv_clear_measured_bw_cache(void)
 -{
 -  if (mbw_cache) {
 -    /* Free the map and all entries */
 -    digestmap_free(mbw_cache, tor_free_);
 -    mbw_cache = NULL;
 -  }
 -}
 -
 -/** Scan the measured bandwidth cache and remove expired entries */
 -STATIC void
 -dirserv_expire_measured_bw_cache(time_t now)
 -{
 -
 -  if (mbw_cache) {
 -    /* Iterate through the cache and check each entry */
 -    DIGESTMAP_FOREACH_MODIFY(mbw_cache, k, mbw_cache_entry_t *, e) {
 -      if (now > e->as_of + MAX_MEASUREMENT_AGE) {
 -        tor_free(e);
 -        MAP_DEL_CURRENT(k);
 -      }
 -    } DIGESTMAP_FOREACH_END;
 -
 -    /* Check if we cleared the whole thing and free if so */
 -    if (digestmap_size(mbw_cache) == 0) {
 -      digestmap_free(mbw_cache, tor_free_);
 -      mbw_cache = 0;
 -    }
 -  }
 -}
 -
 -/** Query the cache by identity digest, return value indicates whether
 - * we found it. The bw_out and as_of_out pointers receive the cached
 - * bandwidth value and the time it was cached if not NULL. */
 -int
 -dirserv_query_measured_bw_cache_kb(const char *node_id, long *bw_kb_out,
 -                                   time_t *as_of_out)
 -{
 -  mbw_cache_entry_t *v = NULL;
 -  int rv = 0;
 -
 -  if (mbw_cache && node_id) {
 -    v = digestmap_get(mbw_cache, node_id);
 -    if (v) {
 -      /* Found something */
 -      rv = 1;
 -      if (bw_kb_out) *bw_kb_out = v->mbw_kb;
 -      if (as_of_out) *as_of_out = v->as_of;
 -    }
 -  }
 -
 -  return rv;
 -}
 -
 -/** Predicate wrapper for dirserv_query_measured_bw_cache() */
 -int
 -dirserv_has_measured_bw(const char *node_id)
 -{
 -  return dirserv_query_measured_bw_cache_kb(node_id, NULL, NULL);
 -}
 -
 -/** Get the current size of the measured bandwidth cache */
 -int
 -dirserv_get_measured_bw_cache_size(void)
 -{
 -  if (mbw_cache) return digestmap_size(mbw_cache);
 -  else return 0;
 -}
 -
 -/** Return the bandwidth we believe for assigning flags; prefer measured
 - * over advertised, and if we have above a threshold quantity of measured
 - * bandwidths, we don't want to ever give flags to unmeasured routers, so
 - * return 0. */
 -static uint32_t
 -dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri)
 -{
 -  int threshold;
 -  uint32_t bw_kb = 0;
 -  long mbw_kb;
 -
 -  tor_assert(ri);
 -  /* Check if we have a measured bandwidth, and check the threshold if not */
 -  if (!(dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
 -                                       &mbw_kb, NULL))) {
 -    threshold = get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
 -    if (routers_with_measured_bw > threshold) {
 -      /* Return zero for unmeasured bandwidth if we are above threshold */
 -      bw_kb = 0;
 -    } else {
 -      /* Return an advertised bandwidth otherwise */
 -      bw_kb = router_get_advertised_bandwidth_capped(ri) / 1000;
 -    }
 -  } else {
 -    /* We have the measured bandwidth in mbw */
 -    bw_kb = (uint32_t)mbw_kb;
 -  }
 -
 -  return bw_kb;
 -}
 -
 -/** Give a statement of our current performance thresholds for inclusion
 - * in a vote document. */
 -char *
 -dirserv_get_flag_thresholds_line(void)
 -{
 -  char *result=NULL;
 -  const int measured_threshold =
 -    get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
 -  const int enough_measured_bw = routers_with_measured_bw > measured_threshold;
 -
 -  tor_asprintf(&result,
 -      "stable-uptime=%lu stable-mtbf=%lu "
 -      "fast-speed=%lu "
 -      "guard-wfu=%.03f%% guard-tk=%lu "
 -      "guard-bw-inc-exits=%lu guard-bw-exc-exits=%lu "
 -      "enough-mtbf=%d ignoring-advertised-bws=%d",
 -      (unsigned long)stable_uptime,
 -      (unsigned long)stable_mtbf,
 -      (unsigned long)fast_bandwidth_kb*1000,
 -      guard_wfu*100,
 -      (unsigned long)guard_tk,
 -      (unsigned long)guard_bandwidth_including_exits_kb*1000,
 -      (unsigned long)guard_bandwidth_excluding_exits_kb*1000,
 -      enough_mtbf_info ? 1 : 0,
 -      enough_measured_bw ? 1 : 0);
 -
 -  return result;
 -}
 -
 -/** Helper: write the router-status information in <b>rs</b> into a newly
 - * allocated character buffer.  Use the same format as in network-status
 - * documents.  If <b>version</b> is non-NULL, add a "v" line for the platform.
 - *
 - * consensus_method is the current consensus method when format is
 - * NS_V3_CONSENSUS or NS_V3_CONSENSUS_MICRODESC. It is ignored for other
 - * formats: pass ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD.
 - *
 - * Return 0 on success, -1 on failure.
 - *
 - * The format argument has one of the following values:
 - *   NS_V2 - Output an entry suitable for a V2 NS opinion document
 - *   NS_V3_CONSENSUS - Output the first portion of a V3 NS consensus entry
 - *        for consensus_method.
 - *   NS_V3_CONSENSUS_MICRODESC - Output the first portion of a V3 microdesc
 - *        consensus entry for consensus_method.
 - *   NS_V3_VOTE - Output a complete V3 NS vote. If <b>vrs</b> is present,
 - *        it contains additional information for the vote.
 - *   NS_CONTROL_PORT - Output a NS document for the control port.
 - */
 -char *
 -routerstatus_format_entry(const routerstatus_t *rs, const char *version,
 -                          const char *protocols,
 -                          routerstatus_format_type_t format,
 -                          int consensus_method,
 -                          const vote_routerstatus_t *vrs)
 -{
 -  char *summary;
 -  char *result = NULL;
 -
 -  char published[ISO_TIME_LEN+1];
 -  char identity64[BASE64_DIGEST_LEN+1];
 -  char digest64[BASE64_DIGEST_LEN+1];
 -  smartlist_t *chunks = smartlist_new();
 -
 -  format_iso_time(published, rs->published_on);
 -  digest_to_base64(identity64, rs->identity_digest);
 -  digest_to_base64(digest64, rs->descriptor_digest);
 -
 -  smartlist_add_asprintf(chunks,
 -                   "r %s %s %s%s%s %s %d %d\n",
 -                   rs->nickname,
 -                   identity64,
 -                   (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64,
 -                   (format==NS_V3_CONSENSUS_MICRODESC)?"":" ",
 -                   published,
 -                   fmt_addr32(rs->addr),
 -                   (int)rs->or_port,
 -                   (int)rs->dir_port);
 -
 -  /* TODO: Maybe we want to pass in what we need to build the rest of
 -   * this here, instead of in the caller. Then we could use the
 -   * networkstatus_type_t values, with an additional control port value
 -   * added -MP */
 -
 -  /* V3 microdesc consensuses only have "a" lines in later consensus methods
 -   */
 -  if (format == NS_V3_CONSENSUS_MICRODESC &&
 -      consensus_method < MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS)
 -    goto done;
 -
 -  /* Possible "a" line. At most one for now. */
 -  if (!tor_addr_is_null(&rs->ipv6_addr)) {
 -    smartlist_add_asprintf(chunks, "a %s\n",
 -                           fmt_addrport(&rs->ipv6_addr, rs->ipv6_orport));
 -  }
 -
 -  if (format == NS_V3_CONSENSUS || format == NS_V3_CONSENSUS_MICRODESC)
 -    goto done;
 -
 -  smartlist_add_asprintf(chunks,
 -                   "s%s%s%s%s%s%s%s%s%s%s\n",
 -                  /* These must stay in alphabetical order. */
 -                   rs->is_authority?" Authority":"",
 -                   rs->is_bad_exit?" BadExit":"",
 -                   rs->is_exit?" Exit":"",
 -                   rs->is_fast?" Fast":"",
 -                   rs->is_possible_guard?" Guard":"",
 -                   rs->is_hs_dir?" HSDir":"",
 -                   rs->is_flagged_running?" Running":"",
 -                   rs->is_stable?" Stable":"",
 -                   rs->is_v2_dir?" V2Dir":"",
 -                   rs->is_valid?" Valid":"");
 -
 -  /* length of "opt v \n" */
 -#define V_LINE_OVERHEAD 7
 -  if (version && strlen(version) < MAX_V_LINE_LEN - V_LINE_OVERHEAD) {
 -    smartlist_add_asprintf(chunks, "v %s\n", version);
 -  }
 -  if (protocols) {
 -    smartlist_add_asprintf(chunks, "pr %s\n", protocols);
 -  }
 -
 -  if (format != NS_V2) {
 -    const routerinfo_t* desc = router_get_by_id_digest(rs->identity_digest);
 -    uint32_t bw_kb;
 -
 -    if (format != NS_CONTROL_PORT) {
 -      /* Blow up more or less nicely if we didn't get anything or not the
 -       * thing we expected.
 -       */
 -      if (!desc) {
 -        char id[HEX_DIGEST_LEN+1];
 -        char dd[HEX_DIGEST_LEN+1];
 -
 -        base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN);
 -        base16_encode(dd, sizeof(dd), rs->descriptor_digest, DIGEST_LEN);
 -        log_warn(LD_BUG, "Cannot get any descriptor for %s "
 -            "(wanted descriptor %s).",
 -            id, dd);
 -        goto err;
 -      }
 -
 -      /* This assert could fire for the control port, because
 -       * it can request NS documents before all descriptors
 -       * have been fetched. Therefore, we only do this test when
 -       * format != NS_CONTROL_PORT. */
 -      if (tor_memneq(desc->cache_info.signed_descriptor_digest,
 -            rs->descriptor_digest,
 -            DIGEST_LEN)) {
 -        char rl_d[HEX_DIGEST_LEN+1];
 -        char rs_d[HEX_DIGEST_LEN+1];
 -        char id[HEX_DIGEST_LEN+1];
 -
 -        base16_encode(rl_d, sizeof(rl_d),
 -            desc->cache_info.signed_descriptor_digest, DIGEST_LEN);
 -        base16_encode(rs_d, sizeof(rs_d), rs->descriptor_digest, DIGEST_LEN);
 -        base16_encode(id, sizeof(id), rs->identity_digest, DIGEST_LEN);
 -        log_err(LD_BUG, "descriptor digest in routerlist does not match "
 -            "the one in routerstatus: %s vs %s "
 -            "(router %s)\n",
 -            rl_d, rs_d, id);
 -
 -        tor_assert(tor_memeq(desc->cache_info.signed_descriptor_digest,
 -              rs->descriptor_digest,
 -              DIGEST_LEN));
 -      }
 -    }
 -
 -    if (format == NS_CONTROL_PORT && rs->has_bandwidth) {
 -      bw_kb = rs->bandwidth_kb;
 -    } else {
 -      tor_assert(desc);
 -      bw_kb = router_get_advertised_bandwidth_capped(desc) / 1000;
 -    }
 -    smartlist_add_asprintf(chunks,
 -                     "w Bandwidth=%d", bw_kb);
 -
 -    if (format == NS_V3_VOTE && vrs && vrs->has_measured_bw) {
 -      smartlist_add_asprintf(chunks,
 -                       " Measured=%d", vrs->measured_bw_kb);
 -    }
 -    /* Write down guardfraction information if we have it. */
 -    if (format == NS_V3_VOTE && vrs && vrs->status.has_guardfraction) {
 -      smartlist_add_asprintf(chunks,
 -                             " GuardFraction=%d",
 -                             vrs->status.guardfraction_percentage);
 -    }
 -
 -    smartlist_add_strdup(chunks, "\n");
 -
 -    if (desc) {
 -      summary = policy_summarize(desc->exit_policy, AF_INET);
 -      smartlist_add_asprintf(chunks, "p %s\n", summary);
 -      tor_free(summary);
 -    }
 -
 -    if (format == NS_V3_VOTE && vrs) {
 -      if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
 -        smartlist_add_strdup(chunks, "id ed25519 none\n");
 -      } else {
 -        char ed_b64[BASE64_DIGEST256_LEN+1];
 -        digest256_to_base64(ed_b64, (const char*)vrs->ed25519_id);
 -        smartlist_add_asprintf(chunks, "id ed25519 %s\n", ed_b64);
 -      }
 -    }
 -  }
 -
 - done:
 -  result = smartlist_join_strings(chunks, "", 0, NULL);
 -
 - err:
 -  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
 -  smartlist_free(chunks);
 -
 -  return result;
 -}
 -
 -/** Extract status information from <b>ri</b> and from other authority
 - * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
 - * set.
 - *
 - * We assume that ri-\>is_running has already been set, e.g. by
 - *   dirserv_set_router_is_running(ri, now);
 - */
 -void
 -set_routerstatus_from_routerinfo(routerstatus_t *rs,
 -                                 node_t *node,
 -                                 routerinfo_t *ri,
 -                                 time_t now,
 -                                 int listbadexits)
 -{
 -  const or_options_t *options = get_options();
 -  uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri);
 -
 -  memset(rs, 0, sizeof(routerstatus_t));
 -
 -  rs->is_authority =
 -    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
 -
 -  /* Already set by compute_performance_thresholds. */
 -  rs->is_exit = node->is_exit;
 -  rs->is_stable = node->is_stable =
 -    !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
 -  rs->is_fast = node->is_fast =
 -    !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
 -  rs->is_flagged_running = node->is_running; /* computed above */
 -
 -  rs->is_valid = node->is_valid;
 -
 -  if (node->is_fast && node->is_stable &&
 -      ri->supports_tunnelled_dir_requests &&
 -      ((options->AuthDirGuardBWGuarantee &&
 -        routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) ||
 -       routerbw_kb >= MIN(guard_bandwidth_including_exits_kb,
 -                          guard_bandwidth_excluding_exits_kb))) {
 -    long tk = rep_hist_get_weighted_time_known(
 -                                      node->identity, now);
 -    double wfu = rep_hist_get_weighted_fractional_uptime(
 -                                      node->identity, now);
 -    rs->is_possible_guard = (wfu >= guard_wfu && tk >= guard_tk) ? 1 : 0;
 -  } else {
 -    rs->is_possible_guard = 0;
 -  }
 -
 -  rs->is_bad_exit = listbadexits && node->is_bad_exit;
 -  rs->is_hs_dir = node->is_hs_dir =
 -    dirserv_thinks_router_is_hs_dir(ri, node, now);
 -
 -  rs->is_named = rs->is_unnamed = 0;
 -
 -  rs->published_on = ri->cache_info.published_on;
 -  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
 -  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
 -         DIGEST_LEN);
 -  rs->addr = ri->addr;
 -  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
 -  rs->or_port = ri->or_port;
 -  rs->dir_port = ri->dir_port;
 -  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
 -  if (options->AuthDirHasIPv6Connectivity == 1 &&
 -      !tor_addr_is_null(&ri->ipv6_addr) &&
 -      node->last_reachable6 >= now - REACHABLE_TIMEOUT) {
 -    /* We're configured as having IPv6 connectivity. There's an IPv6
 -       OR port and it's reachable so copy it to the routerstatus.  */
 -    tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
 -    rs->ipv6_orport = ri->ipv6_orport;
 -  } else {
 -    tor_addr_make_null(&rs->ipv6_addr, AF_INET6);
 -    rs->ipv6_orport = 0;
 -  }
 -
 -  if (options->TestingTorNetwork) {
 -    dirserv_set_routerstatus_testing(rs);
 -  }
 -}
 -
 -/** Use TestingDirAuthVoteExit, TestingDirAuthVoteGuard, and
 - * TestingDirAuthVoteHSDir to give out the Exit, Guard, and HSDir flags,
 - * respectively. But don't set the corresponding node flags.
 - * Should only be called if TestingTorNetwork is set. */
 -STATIC void
 -dirserv_set_routerstatus_testing(routerstatus_t *rs)
 -{
 -  const or_options_t *options = get_options();
 -
 -  tor_assert(options->TestingTorNetwork);
 -
 -  if (routerset_contains_routerstatus(options->TestingDirAuthVoteExit,
 -                                      rs, 0)) {
 -    rs->is_exit = 1;
 -  } else if (options->TestingDirAuthVoteExitIsStrict) {
 -    rs->is_exit = 0;
 -  }
 -
 -  if (routerset_contains_routerstatus(options->TestingDirAuthVoteGuard,
 -                                      rs, 0)) {
 -    rs->is_possible_guard = 1;
 -  } else if (options->TestingDirAuthVoteGuardIsStrict) {
 -    rs->is_possible_guard = 0;
 -  }
 -
 -  if (routerset_contains_routerstatus(options->TestingDirAuthVoteHSDir,
 -                                      rs, 0)) {
 -    rs->is_hs_dir = 1;
 -  } else if (options->TestingDirAuthVoteHSDirIsStrict) {
 -    rs->is_hs_dir = 0;
 -  }
 -}
 -
 -/** The guardfraction of the guard with identity fingerprint <b>guard_id</b>
 - *  is <b>guardfraction_percentage</b>. See if we have a vote routerstatus for
 - *  this guard in <b>vote_routerstatuses</b>, and if we do, register the
 - *  information to it.
 - *
 - *  Return 1 if we applied the information and 0 if we couldn't find a
 - *  matching guard.
 - *
 - * Requires that <b>vote_routerstatuses</b> be sorted.
 - */
 -static int
 -guardfraction_line_apply(const char *guard_id,
 -                      uint32_t guardfraction_percentage,
 -                      smartlist_t *vote_routerstatuses)
 -{
 -  vote_routerstatus_t *vrs = NULL;
 -
 -  tor_assert(vote_routerstatuses);
 -
 -  vrs = smartlist_bsearch(vote_routerstatuses, guard_id,
 -                         compare_digest_to_vote_routerstatus_entry);
 -
 -  if (!vrs) {
 -    return 0;
 -  }
 -
 -  vrs->status.has_guardfraction = 1;
 -  vrs->status.guardfraction_percentage = guardfraction_percentage;
 -
 -  return 1;
 -}
 -
 -/* Given a guard line from a guardfraction file, parse it and register
 - * its information to <b>vote_routerstatuses</b>.
 - *
 - * Return:
 - * * 1 if the line was proper and its information got registered.
 - * * 0 if the line was proper but no currently active guard was found
 - *     to register the guardfraction information to.
 - * * -1 if the line could not be parsed and set <b>err_msg</b> to a
 -      newly allocated string containing the error message.
 - */
 -static int
 -guardfraction_file_parse_guard_line(const char *guard_line,
 -                                    smartlist_t *vote_routerstatuses,
 -                                    char **err_msg)
 -{
 -  char guard_id[DIGEST_LEN];
 -  uint32_t guardfraction;
 -  char *inputs_tmp = NULL;
 -  int num_ok = 1;
 -
 -  smartlist_t *sl = smartlist_new();
 -  int retval = -1;
 -
 -  tor_assert(err_msg);
 -
 -  /* guard_line should contain something like this:
 -     <hex digest> <guardfraction> <appearances> */
 -  smartlist_split_string(sl, guard_line, " ",
 -                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
 -  if (smartlist_len(sl) < 3) {
 -    tor_asprintf(err_msg, "bad line '%s'", guard_line);
 -    goto done;
 -  }
 -
 -  inputs_tmp = smartlist_get(sl, 0);
 -  if (strlen(inputs_tmp) != HEX_DIGEST_LEN ||
 -      base16_decode(guard_id, DIGEST_LEN,
 -                    inputs_tmp, HEX_DIGEST_LEN) != DIGEST_LEN) {
 -    tor_asprintf(err_msg, "bad digest '%s'", inputs_tmp);
 -    goto done;
 -  }
 -
 -  inputs_tmp = smartlist_get(sl, 1);
 -  /* Guardfraction is an integer in [0, 100]. */
 -  guardfraction =
 -    (uint32_t) tor_parse_long(inputs_tmp, 10, 0, 100, &num_ok, NULL);
 -  if (!num_ok) {
 -    tor_asprintf(err_msg, "wrong percentage '%s'", inputs_tmp);
 -    goto done;
 -  }
 -
 -  /* If routerstatuses were provided, apply this info to actual routers. */
 -  if (vote_routerstatuses) {
 -    retval = guardfraction_line_apply(guard_id, guardfraction,
 -                                      vote_routerstatuses);
 -  } else {
 -    retval = 0; /* If we got this far, line was correctly formatted. */
 -  }
 -
 - done:
 -
 -  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
 -  smartlist_free(sl);
 -
 -  return retval;
 -}
 -
 -/** Given an inputs line from a guardfraction file, parse it and
 - *  register its information to <b>total_consensuses</b> and
 - *  <b>total_days</b>.
 - *
 - *  Return 0 if it parsed well. Return -1 if there was an error, and
 - *  set <b>err_msg</b> to a newly allocated string containing the
 - *  error message.
 - */
 -static int
 -guardfraction_file_parse_inputs_line(const char *inputs_line,
 -                                     int *total_consensuses,
 -                                     int *total_days,
 -                                     char **err_msg)
 -{
 -  int retval = -1;
 -  char *inputs_tmp = NULL;
 -  int num_ok = 1;
 -  smartlist_t *sl = smartlist_new();
 -
 -  tor_assert(err_msg);
 -
 -  /* Second line is inputs information:
 -   *   n-inputs <total_consensuses> <total_days>. */
 -  smartlist_split_string(sl, inputs_line, " ",
 -                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
 -  if (smartlist_len(sl) < 2) {
 -    tor_asprintf(err_msg, "incomplete line '%s'", inputs_line);
 -    goto done;
 -  }
 -
 -  inputs_tmp = smartlist_get(sl, 0);
 -  *total_consensuses =
 -    (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL);
 -  if (!num_ok) {
 -    tor_asprintf(err_msg, "unparseable consensus '%s'", inputs_tmp);
 -    goto done;
 -  }
 -
 -  inputs_tmp = smartlist_get(sl, 1);
 -  *total_days =
 -    (int) tor_parse_long(inputs_tmp, 10, 0, INT_MAX, &num_ok, NULL);
 -  if (!num_ok) {
 -    tor_asprintf(err_msg, "unparseable days '%s'", inputs_tmp);
 -    goto done;
 -  }
 -
 -  retval = 0;
 -
 - done:
 -  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
 -  smartlist_free(sl);
 -
 -  return retval;
 -}
 -
 -/* Maximum age of a guardfraction file that we are willing to accept. */
 -#define MAX_GUARDFRACTION_FILE_AGE (7*24*60*60) /* approx a week */
 -
 -/** Static strings of guardfraction files. */
 -#define GUARDFRACTION_DATE_STR "written-at"
 -#define GUARDFRACTION_INPUTS "n-inputs"
 -#define GUARDFRACTION_GUARD "guard-seen"
 -#define GUARDFRACTION_VERSION "guardfraction-file-version"
 -
 -/** Given a guardfraction file in a string, parse it and register the
 - *  guardfraction information to the provided vote routerstatuses.
 - *
 - *  This is the rough format of the guardfraction file:
 - *
 - *      guardfraction-file-version 1
 - *      written-at <date and time>
 - *      n-inputs <number of consesuses parsed> <number of days considered>
 - *
 - *      guard-seen <fpr 1> <guardfraction percentage> <consensus appearances>
 - *      guard-seen <fpr 2> <guardfraction percentage> <consensus appearances>
 - *      guard-seen <fpr 3> <guardfraction percentage> <consensus appearances>
 - *      guard-seen <fpr 4> <guardfraction percentage> <consensus appearances>
 - *      guard-seen <fpr 5> <guardfraction percentage> <consensus appearances>
 - *      ...
 - *
 - *  Return -1 if the parsing failed and 0 if it went smoothly. Parsing
 - *  should tolerate errors in all lines but the written-at header.
 - */
 -STATIC int
 -dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
 -                                      smartlist_t *vote_routerstatuses)
 -{
 -  config_line_t *front=NULL, *line;
 -  int ret_tmp;
 -  int retval = -1;
 -  int current_line_n = 0; /* line counter for better log messages */
 -
 -  /* Guardfraction info to be parsed */
 -  int total_consensuses = 0;
 -  int total_days = 0;
 -
 -  /* Stats */
 -  int guards_read_n = 0;
 -  int guards_applied_n = 0;
 -
 -  /* Parse file and split it in lines */
 -  ret_tmp = config_get_lines(guardfraction_file_str, &front, 0);
 -  if (ret_tmp < 0) {
 -    log_warn(LD_CONFIG, "Error reading from guardfraction file");
 -    goto done;
 -  }
 -
 -  /* Sort routerstatuses (needed later when applying guardfraction info) */
 -  if (vote_routerstatuses)
 -    smartlist_sort(vote_routerstatuses, compare_vote_routerstatus_entries);
 -
 -  for (line = front; line; line=line->next) {
 -    current_line_n++;
 -
 -    if (!strcmp(line->key, GUARDFRACTION_VERSION)) {
 -      int num_ok = 1;
 -      unsigned int version;
 -
 -      version =
 -        (unsigned int) tor_parse_long(line->value,
 -                                      10, 0, INT_MAX, &num_ok, NULL);
 -
 -      if (!num_ok || version != 1) {
 -        log_warn(LD_GENERAL, "Got unknown guardfraction version %d.", version);
 -        goto done;
 -      }
 -    } else if (!strcmp(line->key, GUARDFRACTION_DATE_STR)) {
 -      time_t file_written_at;
 -      time_t now = time(NULL);
 -
 -      /* First line is 'written-at <date>' */
 -      if (parse_iso_time(line->value, &file_written_at) < 0) {
 -        log_warn(LD_CONFIG, "Guardfraction:%d: Bad date '%s'. Ignoring",
 -                 current_line_n, line->value);
 -        goto done; /* don't tolerate failure here. */
 -      }
 -      if (file_written_at < now - MAX_GUARDFRACTION_FILE_AGE) {
 -        log_warn(LD_CONFIG, "Guardfraction:%d: was written very long ago '%s'",
 -                 current_line_n, line->value);
 -        goto done; /* don't tolerate failure here. */
 -      }
 -    } else if (!strcmp(line->key, GUARDFRACTION_INPUTS)) {
 -      char *err_msg = NULL;
 -
 -      if (guardfraction_file_parse_inputs_line(line->value,
 -                                               &total_consensuses,
 -                                               &total_days,
 -                                               &err_msg) < 0) {
 -        log_warn(LD_CONFIG, "Guardfraction:%d: %s",
 -                 current_line_n, err_msg);
 -        tor_free(err_msg);
 -        continue;
 -      }
 -
 -    } else if (!strcmp(line->key, GUARDFRACTION_GUARD)) {
 -      char *err_msg = NULL;
 -
 -      ret_tmp = guardfraction_file_parse_guard_line(line->value,
 -                                                    vote_routerstatuses,
 -                                                    &err_msg);
 -      if (ret_tmp < 0) { /* failed while parsing the guard line */
 -        log_warn(LD_CONFIG, "Guardfraction:%d: %s",
 -                 current_line_n, err_msg);
 -        tor_free(err_msg);
 -        continue;
 -      }
 -
 -      /* Successfully parsed guard line. Check if it was applied properly. */
 -      guards_read_n++;
 -      if (ret_tmp > 0) {
 -        guards_applied_n++;
 -      }
 -    } else {
 -      log_warn(LD_CONFIG, "Unknown guardfraction line %d (%s %s)",
 -               current_line_n, line->key, line->value);
 -    }
 -  }
 -
 -  retval = 0;
 -
 -  log_info(LD_CONFIG,
 -           "Successfully parsed guardfraction file with %d consensuses over "
 -           "%d days. Parsed %d nodes and applied %d of them%s.",
 -           total_consensuses, total_days, guards_read_n, guards_applied_n,
 -           vote_routerstatuses ? "" : " (no routerstatus provided)" );
 -
 - done:
 -  config_free_lines(front);
 -
 -  if (retval < 0) {
 -    return retval;
 -  } else {
 -    return guards_read_n;
 -  }
 +  return 1;
  }
  
 -/** Read a guardfraction file at <b>fname</b> and load all its
 - *  information to <b>vote_routerstatuses</b>. */
 +/** Return 1 if we should fetch new networkstatuses, descriptors, etc
 + * on the "mirror" schedule rather than the "client" schedule.
 + */
  int
 -dirserv_read_guardfraction_file(const char *fname,
 -                             smartlist_t *vote_routerstatuses)
 +directory_fetches_dir_info_early(const or_options_t *options)
  {
 -  char *guardfraction_file_str;
 -
 -  /* Read file to a string */
 -  guardfraction_file_str = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
 -  if (!guardfraction_file_str) {
 -      log_warn(LD_FS, "Cannot open guardfraction file '%s'. Failing.", fname);
 -      return -1;
 -  }
 -
 -  return dirserv_read_guardfraction_file_from_str(guardfraction_file_str,
 -                                               vote_routerstatuses);
 +  return directory_fetches_from_authorities(options);
  }
  
 -/**
 - * Helper function to parse out a line in the measured bandwidth file
 - * into a measured_bw_line_t output structure.
 - *
 - * If <b>line_is_after_headers</b> is true, then if we encounter an incomplete
 - * bw line, return -1 and warn, since we are after the headers and we should
 - * only parse bw lines. Return 0 otherwise.
 - *
 - * If <b>line_is_after_headers</b> is false then it means that we are not past
 - * the header block yet. If we encounter an incomplete bw line, return -1 but
 - * don't warn since there could be additional header lines coming. If we
 - * encounter a proper bw line, return 0 (and we got past the headers).
 +/** Return 1 if we should fetch new networkstatuses, descriptors, etc
 + * on a very passive schedule -- waiting long enough for ordinary clients
 + * to probably have the info we want. These would include bridge users,
 + * and maybe others in the future e.g. if a Tor client uses another Tor
 + * client as a directory guard.
   */
 -STATIC int
 -measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line,
 -                       int line_is_after_headers)
 +int
 +directory_fetches_dir_info_later(const or_options_t *options)
  {
 -  char *line = tor_strdup(orig_line);
 -  char *cp = line;
 -  int got_bw = 0;
 -  int got_node_id = 0;
 -  char *strtok_state; /* lame sauce d'jour */
 -
 -  if (strlen(line) == 0) {
 -    log_warn(LD_DIRSERV, "Empty line in bandwidth file");
 -    tor_free(line);
 -    return -1;
 -  }
 -
 -  /* Remove end of line character, so that is not part of the token */
 -  if (line[strlen(line) - 1] == '\n') {
 -    line[strlen(line) - 1] = '\0';
 -  }
 -
 -  cp = tor_strtok_r(cp, " \t", &strtok_state);
 -
 -  if (!cp) {
 -    log_warn(LD_DIRSERV, "Invalid line in bandwidth file: %s",
 -             escaped(orig_line));
 -    tor_free(line);
 -    return -1;
 -  }
 -
 -  if (orig_line[strlen(orig_line)-1] != '\n') {
 -    log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s",
 -             escaped(orig_line));
 -    tor_free(line);
 -    return -1;
 -  }
 -
 -  do {
 -    if (strcmpstart(cp, "bw=") == 0) {
 -      int parse_ok = 0;
 -      char *endptr;
 -      if (got_bw) {
 -        log_warn(LD_DIRSERV, "Double bw= in bandwidth file line: %s",
 -                 escaped(orig_line));
 -        tor_free(line);
 -        return -1;
 -      }
 -      cp+=strlen("bw=");
 -
 -      out->bw_kb = tor_parse_long(cp, 10, 0, LONG_MAX, &parse_ok, &endptr);
 -      if (!parse_ok || (*endptr && !TOR_ISSPACE(*endptr))) {
 -        log_warn(LD_DIRSERV, "Invalid bandwidth in bandwidth file line: %s",
 -                 escaped(orig_line));
 -        tor_free(line);
 -        return -1;
 -      }
 -      got_bw=1;
 -    } else if (strcmpstart(cp, "node_id=$") == 0) {
 -      if (got_node_id) {
 -        log_warn(LD_DIRSERV, "Double node_id= in bandwidth file line: %s",
 -                 escaped(orig_line));
 -        tor_free(line);
 -        return -1;
 -      }
 -      cp+=strlen("node_id=$");
 -
 -      if (strlen(cp) != HEX_DIGEST_LEN ||
 -          base16_decode(out->node_id, DIGEST_LEN,
 -                        cp, HEX_DIGEST_LEN) != DIGEST_LEN) {
 -        log_warn(LD_DIRSERV, "Invalid node_id in bandwidth file line: %s",
 -                 escaped(orig_line));
 -        tor_free(line);
 -        return -1;
 -      }
 -      strlcpy(out->node_hex, cp, sizeof(out->node_hex));
 -      got_node_id=1;
 -    }
 -  } while ((cp = tor_strtok_r(NULL, " \t", &strtok_state)));
 +  return options->UseBridges != 0;
 +}
  
 -  if (got_bw && got_node_id) {
 -    tor_free(line);
 -    return 0;
 -  } else if (line_is_after_headers == 0) {
 -    /* There could be additional header lines, therefore do not give warnings
 -     * but returns -1 since it's not a complete bw line. */
 -    log_debug(LD_DIRSERV, "Missing bw or node_id in bandwidth file line: %s",
 -             escaped(orig_line));
 -    tor_free(line);
 -    return -1;
 -  } else {
 -    log_warn(LD_DIRSERV, "Incomplete line in bandwidth file: %s",
 -             escaped(orig_line));
 -    tor_free(line);
 -    return -1;
 -  }
 +/** Return true iff we want to serve certificates for authorities
 + * that we don't acknowledge as authorities ourself.
 + * Use we_want_to_fetch_unknown_auth_certs to check if we want to fetch
 + * and keep these certificates.
 + */
 +int
 +directory_caches_unknown_auth_certs(const or_options_t *options)
 +{
 +  return dir_server_mode(options) || options->BridgeRelay;
  }
  
 -/**
 - * Helper function to apply a parsed measurement line to a list
 - * of bandwidth statuses. Returns true if a line is found,
 - * false otherwise.
 +/** Return 1 if we want to fetch and serve descriptors, networkstatuses, etc
 + * Else return 0.
 + * Check options->DirPort_set and directory_permits_begindir_requests()
 + * to see if we are willing to serve these directory documents to others via
 + * the DirPort and begindir-over-ORPort, respectively.
 + *
 + * To check if we should fetch documents, use we_want_to_fetch_flavor and
 + * we_want_to_fetch_unknown_auth_certs instead of this function.
   */
 -STATIC int
 -measured_bw_line_apply(measured_bw_line_t *parsed_line,
 -                       smartlist_t *routerstatuses)
 +int
 +directory_caches_dir_info(const or_options_t *options)
  {
 -  vote_routerstatus_t *rs = NULL;
 -  if (!routerstatuses)
 +  if (options->BridgeRelay || dir_server_mode(options))
 +    return 1;
 +  if (!server_mode(options) || !advertised_server_mode())
      return 0;
 +  /* We need an up-to-date view of network info if we're going to try to
 +   * block exit attempts from unknown relays. */
 +  return ! router_my_exit_policy_is_reject_star() &&
 +    should_refuse_unknown_exits(options);
 +}
  
 -  rs = smartlist_bsearch(routerstatuses, parsed_line->node_id,
 -                         compare_digest_to_vote_routerstatus_entry);
 -
 -  if (rs) {
 -    rs->has_measured_bw = 1;
 -    rs->measured_bw_kb = (uint32_t)parsed_line->bw_kb;
 -  } else {
 -    log_info(LD_DIRSERV, "Node ID %s not found in routerstatus list",
 -             parsed_line->node_hex);
 -  }
 -
 -  return rs != NULL;
 +/** Return 1 if we want to allow remote clients to ask us directory
 + * requests via the "begin_dir" interface, which doesn't require
 + * having any separate port open. */
 +int
 +directory_permits_begindir_requests(const or_options_t *options)
 +{
 +  return options->BridgeRelay != 0 || dir_server_mode(options);
  }
  
 -/**
 - * Read the measured bandwidth list file, apply it to the list of
 - * vote_routerstatus_t and store all the headers in <b>bw_file_headers</b>.
 - * Returns -1 on error, 0 otherwise.
 +/** Return 1 if we have no need to fetch new descriptors. This generally
 + * happens when we're not a dir cache and we haven't built any circuits
 + * lately.
   */
  int
 -dirserv_read_measured_bandwidths(const char *from_file,
 -                                 smartlist_t *routerstatuses,
 -                                 smartlist_t *bw_file_headers)
 +directory_too_idle_to_fetch_descriptors(const or_options_t *options,
 +                                        time_t now)
  {
 -  FILE *fp = tor_fopen_cloexec(from_file, "r");
 -  int applied_lines = 0;
 -  time_t file_time, now;
 -  int ok;
 -   /* This flag will be 1 only when the first successful bw measurement line
 -   * has been encountered, so that measured_bw_line_parse don't give warnings
 -   * if there are additional header lines, as introduced in Bandwidth List spec
 -   * version 1.1.0 */
 -  int line_is_after_headers = 0;
 -  int rv = -1;
 -  char *line = NULL;
 -  size_t n = 0;
 -
 -  /* Initialise line, so that we can't possibly run off the end. */
 -
 -  if (fp == NULL) {
 -    log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s",
 -             from_file);
 -    goto err;
 -  }
 +  return !directory_caches_dir_info(options) &&
 +         !options->FetchUselessDescriptors &&
 +         rep_hist_circbuilding_dormant(now);
 +}
  
 -  /* If fgets fails, line is either unmodified, or indeterminate. */
 -  if (tor_getline(&line,&n,fp) <= 0) {
 -    log_warn(LD_DIRSERV, "Empty bandwidth file");
 -    goto err;
 -  }
 +/********************************************************************/
  
 -  if (!strlen(line) || line[strlen(line)-1] != '\n') {
 -    log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s",
 -             escaped(line));
 -    goto err;
 -  }
 +/** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
 + * currently serving. */
 +static strmap_t *cached_consensuses = NULL;
  
 -  line[strlen(line)-1] = '\0';
 -  file_time = (time_t)tor_parse_ulong(line, 10, 0, ULONG_MAX, &ok, NULL);
 -  if (!ok) {
 -    log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s",
 -             escaped(line));
 -    goto err;
 -  }
 +/** Decrement the reference count on <b>d</b>, and free it if it no longer has
 + * any references. */
 +void
 +cached_dir_decref(cached_dir_t *d)
 +{
 +  if (!d || --d->refcnt > 0)
 +    return;
 +  clear_cached_dir(d);
 +  tor_free(d);
 +}
  
 -  now = time(NULL);
 -  if ((now - file_time) > MAX_MEASUREMENT_AGE) {
 -    log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u",
 -             (unsigned)(time(NULL) - file_time));
 -    goto err;
 +/** Allocate and return a new cached_dir_t containing the string <b>s</b>,
 + * published at <b>published</b>. */
 +cached_dir_t *
 +new_cached_dir(char *s, time_t published)
 +{
 +  cached_dir_t *d = tor_malloc_zero(sizeof(cached_dir_t));
 +  d->refcnt = 1;
 +  d->dir = s;
 +  d->dir_len = strlen(s);
 +  d->published = published;
 +  if (tor_compress(&(d->dir_compressed), &(d->dir_compressed_len),
 +                   d->dir, d->dir_len, ZLIB_METHOD)) {
 +    log_warn(LD_BUG, "Error compressing directory");
    }
 +  return d;
 +}
  
 -  /* If timestamp was correct and bw_file_headers is not NULL,
 -   * add timestamp to bw_file_headers */
 -  if (bw_file_headers)
 -    smartlist_add_asprintf(bw_file_headers, "timestamp=%lu",
 -                           (unsigned long)file_time);
 -
 -  if (routerstatuses)
 -    smartlist_sort(routerstatuses, compare_vote_routerstatus_entries);
 -
 -  while (!feof(fp)) {
 -    measured_bw_line_t parsed_line;
 -    if (tor_getline(&line, &n, fp) >= 0) {
 -      if (measured_bw_line_parse(&parsed_line, line,
 -                                 line_is_after_headers) != -1) {
 -        /* This condition will be true when the first complete valid bw line
 -         * has been encountered, which means the end of the header lines. */
 -        line_is_after_headers = 1;
 -        /* Also cache the line for dirserv_get_bandwidth_for_router() */
 -        dirserv_cache_measured_bw(&parsed_line, file_time);
 -        if (measured_bw_line_apply(&parsed_line, routerstatuses) > 0)
 -          applied_lines++;
 -      /* if the terminator is found, it is the end of header lines, set the
 -       * flag but do not store anything */
 -      } else if (strcmp(line, BW_FILE_HEADERS_TERMINATOR) == 0) {
 -        line_is_after_headers = 1;
 -      /* if the line was not a correct relay line nor the terminator and
 -       * the end of the header lines has not been detected yet
 -       * and it is key_value and bw_file_headers did not reach the maximum
 -       * number of headers,
 -       * then assume this line is a header and add it to bw_file_headers */
 -      } else if (bw_file_headers &&
 -              (line_is_after_headers == 0) &&
 -              string_is_key_value(LOG_DEBUG, line) &&
 -              !strchr(line, ' ') &&
 -              (smartlist_len(bw_file_headers)
 -               < MAX_BW_FILE_HEADER_COUNT_IN_VOTE)) {
 -        line[strlen(line)-1] = '\0';
 -        smartlist_add_strdup(bw_file_headers, line);
 -      };
 -    }
 -  }
 +/** Remove all storage held in <b>d</b>, but do not free <b>d</b> itself. */
 +static void
 +clear_cached_dir(cached_dir_t *d)
 +{
 +  tor_free(d->dir);
 +  tor_free(d->dir_compressed);
 +  memset(d, 0, sizeof(cached_dir_t));
 +}
  
 -  /* Now would be a nice time to clean the cache, too */
 -  dirserv_expire_measured_bw_cache(now);
 +/** Free all storage held by the cached_dir_t in <b>d</b>. */
 +static void
 +free_cached_dir_(void *_d)
 +{
 +  cached_dir_t *d;
 +  if (!_d)
 +    return;
  
 -  log_info(LD_DIRSERV,
 -           "Bandwidth measurement file successfully read. "
 -           "Applied %d measurements.", applied_lines);
 -  rv = 0;
 +  d = (cached_dir_t *)_d;
 +  cached_dir_decref(d);
 +}
  
 - err:
 -  if (line) {
 -    // we need to raw_free this buffer because we got it from tor_getdelim()
 -    raw_free(line);
 -  }
 -  if (fp)
 -    fclose(fp);
 -  return rv;
 +/** Replace the v3 consensus networkstatus of type <b>flavor_name</b> that
 + * we're serving with <b>networkstatus</b>, published at <b>published</b>.  No
 + * validation is performed. */
 +void
 +dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
++                                           size_t networkstatus_len,
 +                                           const char *flavor_name,
 +                                           const common_digests_t *digests,
 +                                           const uint8_t *sha3_as_signed,
 +                                           time_t published)
 +{
 +  cached_dir_t *new_networkstatus;
 +  cached_dir_t *old_networkstatus;
 +  if (!cached_consensuses)
 +    cached_consensuses = strmap_new();
 +
-   new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published);
++  new_networkstatus =
++    new_cached_dir(tor_memdup_nulterm(networkstatus, networkstatus_len),
++                   published);
 +  memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t));
 +  memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed,
 +         DIGEST256_LEN);
 +  old_networkstatus = strmap_set(cached_consensuses, flavor_name,
 +                                 new_networkstatus);
 +  if (old_networkstatus)
 +    cached_dir_decref(old_networkstatus);
 +}
 +
 +/** Return the latest downloaded consensus networkstatus in encoded, signed,
 + * optionally compressed format, suitable for sending to clients. */
 +cached_dir_t *
 +dirserv_get_consensus(const char *flavor_name)
 +{
 +  if (!cached_consensuses)
 +    return NULL;
 +  return strmap_get(cached_consensuses, flavor_name);
  }
  
  /** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t
diff --cc src/feature/dirparse/authcert_parse.c
index 2ba46bb8f,000000000..334baf8b1
mode 100644,000000..100644
--- a/src/feature/dirparse/authcert_parse.c
+++ b/src/feature/dirparse/authcert_parse.c
@@@ -1,207 -1,0 +1,209 @@@
 +/* Copyright (c) 2001 Matej Pfajfar.
 + * Copyright (c) 2001-2004, Roger Dingledine.
 + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
 + * Copyright (c) 2007-2018, The Tor Project, Inc. */
 +/* See LICENSE for licensing information */
 +
 +#include "core/or/or.h"
 +#include "feature/dirparse/authcert_parse.h"
 +#include "feature/dirparse/parsecommon.h"
 +#include "feature/dirparse/sigcommon.h"
 +#include "feature/dirparse/unparseable.h"
 +#include "feature/nodelist/authcert.h"
 +#include "lib/memarea/memarea.h"
 +
 +#include "feature/nodelist/authority_cert_st.h"
 +
 +/** List of tokens recognized in V3 authority certificates. */
 +static token_rule_t dir_key_certificate_table[] = {
 +#include "feature/dirparse/authcert_members.i"
 +  T1("fingerprint",      K_FINGERPRINT,              CONCAT_ARGS, NO_OBJ ),
 +  END_OF_TABLE
 +};
 +
 +/** Parse a key certificate from <b>s</b>; point <b>end-of-string</b> to
 + * the first character after the certificate. */
 +authority_cert_t *
- authority_cert_parse_from_string(const char *s, const char **end_of_string)
++authority_cert_parse_from_string(const char *s, size_t maxlen,
++                                 const char **end_of_string)
 +{
 +  /** Reject any certificate at least this big; it is probably an overflow, an
 +   * attack, a bug, or some other nonsense. */
 +#define MAX_CERT_SIZE (128*1024)
 +
 +  authority_cert_t *cert = NULL, *old_cert;
 +  smartlist_t *tokens = NULL;
 +  char digest[DIGEST_LEN];
 +  directory_token_t *tok;
 +  char fp_declared[DIGEST_LEN];
-   char *eos;
++  const char *eos;
 +  size_t len;
 +  int found;
 +  memarea_t *area = NULL;
++  const char *end_of_s = s + maxlen;
 +  const char *s_dup = s;
 +
-   s = eat_whitespace(s);
-   eos = strstr(s, "\ndir-key-certification");
++  s = eat_whitespace_eos(s, end_of_s);
++  eos = tor_memstr(s, end_of_s - s, "\ndir-key-certification");
 +  if (! eos) {
 +    log_warn(LD_DIR, "No signature found on key certificate");
 +    return NULL;
 +  }
-   eos = strstr(eos, "\n-----END SIGNATURE-----\n");
++  eos = tor_memstr(eos, end_of_s - eos, "\n-----END SIGNATURE-----\n");
 +  if (! eos) {
 +    log_warn(LD_DIR, "No end-of-signature found on key certificate");
 +    return NULL;
 +  }
-   eos = strchr(eos+2, '\n');
++  eos = memchr(eos+2, '\n', end_of_s - (eos+2));
 +  tor_assert(eos);
 +  ++eos;
 +  len = eos - s;
 +
 +  if (len > MAX_CERT_SIZE) {
 +    log_warn(LD_DIR, "Certificate is far too big (at %lu bytes long); "
 +             "rejecting", (unsigned long)len);
 +    return NULL;
 +  }
 +
 +  tokens = smartlist_new();
 +  area = memarea_new();
 +  if (tokenize_string(area,s, eos, tokens, dir_key_certificate_table, 0) < 0) {
 +    log_warn(LD_DIR, "Error tokenizing key certificate");
 +    goto err;
 +  }
-   if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version",
++  if (router_get_hash_impl(s, eos - s, digest, "dir-key-certificate-version",
 +                           "\ndir-key-certification", '\n', DIGEST_SHA1) < 0)
 +    goto err;
 +  tok = smartlist_get(tokens, 0);
 +  if (tok->tp != K_DIR_KEY_CERTIFICATE_VERSION || strcmp(tok->args[0], "3")) {
 +    log_warn(LD_DIR,
 +             "Key certificate does not begin with a recognized version (3).");
 +    goto err;
 +  }
 +
 +  cert = tor_malloc_zero(sizeof(authority_cert_t));
 +  memcpy(cert->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
 +
 +  tok = find_by_keyword(tokens, K_DIR_SIGNING_KEY);
 +  tor_assert(tok->key);
 +  cert->signing_key = tok->key;
 +  tok->key = NULL;
 +  if (crypto_pk_get_digest(cert->signing_key, cert->signing_key_digest))
 +    goto err;
 +
 +  tok = find_by_keyword(tokens, K_DIR_IDENTITY_KEY);
 +  tor_assert(tok->key);
 +  cert->identity_key = tok->key;
 +  tok->key = NULL;
 +
 +  tok = find_by_keyword(tokens, K_FINGERPRINT);
 +  tor_assert(tok->n_args);
 +  if (base16_decode(fp_declared, DIGEST_LEN, tok->args[0],
 +                    strlen(tok->args[0])) != DIGEST_LEN) {
 +    log_warn(LD_DIR, "Couldn't decode key certificate fingerprint %s",
 +             escaped(tok->args[0]));
 +    goto err;
 +  }
 +
 +  if (crypto_pk_get_digest(cert->identity_key,
 +                           cert->cache_info.identity_digest))
 +    goto err;
 +
 +  if (tor_memneq(cert->cache_info.identity_digest, fp_declared, DIGEST_LEN)) {
 +    log_warn(LD_DIR, "Digest of certificate key didn't match declared "
 +             "fingerprint");
 +    goto err;
 +  }
 +
 +  tok = find_opt_by_keyword(tokens, K_DIR_ADDRESS);
 +  if (tok) {
 +    struct in_addr in;
 +    char *address = NULL;
 +    tor_assert(tok->n_args);
 +    /* XXX++ use some tor_addr parse function below instead. -RD */
 +    if (tor_addr_port_split(LOG_WARN, tok->args[0], &address,
 +                            &cert->dir_port) < 0 ||
 +        tor_inet_aton(address, &in) == 0) {
 +      log_warn(LD_DIR, "Couldn't parse dir-address in certificate");
 +      tor_free(address);
 +      goto err;
 +    }
 +    cert->addr = ntohl(in.s_addr);
 +    tor_free(address);
 +  }
 +
 +  tok = find_by_keyword(tokens, K_DIR_KEY_PUBLISHED);
 +  if (parse_iso_time(tok->args[0], &cert->cache_info.published_on) < 0) {
 +     goto err;
 +  }
 +  tok = find_by_keyword(tokens, K_DIR_KEY_EXPIRES);
 +  if (parse_iso_time(tok->args[0], &cert->expires) < 0) {
 +     goto err;
 +  }
 +
 +  tok = smartlist_get(tokens, smartlist_len(tokens)-1);
 +  if (tok->tp != K_DIR_KEY_CERTIFICATION) {
 +    log_warn(LD_DIR, "Certificate didn't end with dir-key-certification.");
 +    goto err;
 +  }
 +
 +  /* If we already have this cert, don't bother checking the signature. */
 +  old_cert = authority_cert_get_by_digests(
 +                                     cert->cache_info.identity_digest,
 +                                     cert->signing_key_digest);
 +  found = 0;
 +  if (old_cert) {
 +    /* XXXX We could just compare signed_descriptor_digest, but that wouldn't
 +     * buy us much. */
 +    if (old_cert->cache_info.signed_descriptor_len == len &&
 +        old_cert->cache_info.signed_descriptor_body &&
 +        tor_memeq(s, old_cert->cache_info.signed_descriptor_body, len)) {
 +      log_debug(LD_DIR, "We already checked the signature on this "
 +                "certificate; no need to do so again.");
 +      found = 1;
 +    }
 +  }
 +  if (!found) {
 +    if (check_signature_token(digest, DIGEST_LEN, tok, cert->identity_key, 0,
 +                              "key certificate")) {
 +      goto err;
 +    }
 +
 +    tok = find_by_keyword(tokens, K_DIR_KEY_CROSSCERT);
 +    if (check_signature_token(cert->cache_info.identity_digest,
 +                              DIGEST_LEN,
 +                              tok,
 +                              cert->signing_key,
 +                              CST_NO_CHECK_OBJTYPE,
 +                              "key cross-certification")) {
 +      goto err;
 +    }
 +  }
 +
 +  cert->cache_info.signed_descriptor_len = len;
 +  cert->cache_info.signed_descriptor_body = tor_malloc(len+1);
 +  memcpy(cert->cache_info.signed_descriptor_body, s, len);
 +  cert->cache_info.signed_descriptor_body[len] = 0;
 +  cert->cache_info.saved_location = SAVED_NOWHERE;
 +
 +  if (end_of_string) {
 +    *end_of_string = eat_whitespace(eos);
 +  }
 +  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
 +  smartlist_free(tokens);
 +  if (area) {
 +    DUMP_AREA(area, "authority cert");
 +    memarea_drop_all(area);
 +  }
 +  return cert;
 + err:
 +  dump_desc(s_dup, "authority cert");
 +  authority_cert_free(cert);
 +  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
 +  smartlist_free(tokens);
 +  if (area) {
 +    DUMP_AREA(area, "authority cert");
 +    memarea_drop_all(area);
 +  }
 +  return NULL;
 +}
diff --cc src/feature/dirparse/authcert_parse.h
index f63525e04,000000000..e4e9fec99
mode 100644,000000..100644
--- a/src/feature/dirparse/authcert_parse.h
+++ b/src/feature/dirparse/authcert_parse.h
@@@ -1,18 -1,0 +1,19 @@@
 +/* Copyright (c) 2001 Matej Pfajfar.
 + * Copyright (c) 2001-2004, Roger Dingledine.
 + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
 + * Copyright (c) 2007-2018, The Tor Project, Inc. */
 +/* See LICENSE for licensing information */
 +
 +/**
 + * \file authcert_parse.h
 + * \brief Header file for authcert_parse.c.
 + **/
 +
 +#ifndef TOR_AUTHCERT_PARSE_H
 +#define TOR_AUTHCERT_PARSE_H
 +
 +authority_cert_t *authority_cert_parse_from_string(const char *s,
++                                                   size_t maxlen,
 +                                                   const char **end_of_string);
 +
 +#endif /* !defined(TOR_AUTHCERT_PARSE_H) */
diff --cc src/feature/dirparse/ns_parse.c
index 72299e807,000000000..3fccec154
mode 100644,000000..100644
--- a/src/feature/dirparse/ns_parse.c
+++ b/src/feature/dirparse/ns_parse.c
@@@ -1,1685 -1,0 +1,1696 @@@
 +/* Copyright (c) 2001 Matej Pfajfar.
 + * Copyright (c) 2001-2004, Roger Dingledine.
 + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
 + * Copyright (c) 2007-2018, The Tor Project, Inc. */
 +/* See LICENSE for licensing information */
 +
 +/**
 + * \file routerparse.c
 + * \brief Code to parse and validate consensus documents and votes.
 + */
 +
 +#define NS_PARSE_PRIVATE
 +
 +#include "core/or/or.h"
 +#include "app/config/config.h"
 +#include "core/or/versions.h"
 +#include "feature/client/entrynodes.h"
 +#include "feature/dirauth/dirvote.h"
 +#include "feature/dirparse/authcert_parse.h"
 +#include "feature/dirparse/ns_parse.h"
 +#include "feature/dirparse/parsecommon.h"
 +#include "feature/dirparse/routerparse.h"
 +#include "feature/dirparse/sigcommon.h"
 +#include "feature/dirparse/unparseable.h"
 +#include "feature/hs_common/shared_random_client.h"
 +#include "feature/nodelist/authcert.h"
 +#include "feature/nodelist/describe.h"
 +#include "feature/nodelist/networkstatus.h"
 +#include "feature/nodelist/nickname.h"
 +#include "lib/crypt_ops/crypto_format.h"
 +#include "lib/memarea/memarea.h"
 +
 +#include "feature/dirauth/vote_microdesc_hash_st.h"
 +#include "feature/nodelist/authority_cert_st.h"
 +#include "feature/nodelist/document_signature_st.h"
 +#include "feature/nodelist/networkstatus_st.h"
 +#include "feature/nodelist/networkstatus_voter_info_st.h"
 +#include "feature/nodelist/vote_routerstatus_st.h"
 +
 +#undef log
 +#include <math.h>
 +
 +/** List of tokens recognized in the body part of v3 networkstatus
 + * documents. */
 +static token_rule_t rtrstatus_token_table[] = {
 +  T01("p",                   K_P,               CONCAT_ARGS, NO_OBJ ),
 +  T1( "r",                   K_R,                   GE(7),   NO_OBJ ),
 +  T0N("a",                   K_A,                   GE(1),   NO_OBJ ),
 +  T1( "s",                   K_S,                   ARGS,    NO_OBJ ),
 +  T01("v",                   K_V,               CONCAT_ARGS, NO_OBJ ),
 +  T01("w",                   K_W,                   ARGS,    NO_OBJ ),
 +  T0N("m",                   K_M,               CONCAT_ARGS, NO_OBJ ),
 +  T0N("id",                  K_ID,                  GE(2),   NO_OBJ ),
 +  T01("pr",                  K_PROTO,           CONCAT_ARGS, NO_OBJ ),
 +  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
 +  END_OF_TABLE
 +};
 +
 +/** List of tokens recognized in V3 networkstatus votes. */
 +static token_rule_t networkstatus_token_table[] = {
 +  T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
 +                                                   GE(1),       NO_OBJ ),
 +  T1("vote-status",            K_VOTE_STATUS,      GE(1),       NO_OBJ ),
 +  T1("published",              K_PUBLISHED,        CONCAT_ARGS, NO_OBJ ),
 +  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
 +  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
 +  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
 +  T1("voting-delay",           K_VOTING_DELAY,     GE(2),       NO_OBJ ),
 +  T1("known-flags",            K_KNOWN_FLAGS,      ARGS,        NO_OBJ ),
 +  T01("params",                K_PARAMS,           ARGS,        NO_OBJ ),
 +  T( "fingerprint",            K_FINGERPRINT,      CONCAT_ARGS, NO_OBJ ),
 +  T01("signing-ed25519",       K_SIGNING_CERT_ED,  NO_ARGS ,    NEED_OBJ ),
 +  T01("shared-rand-participate",K_SR_FLAG,         NO_ARGS,     NO_OBJ ),
 +  T0N("shared-rand-commit",    K_COMMIT,           GE(3),       NO_OBJ ),
 +  T01("shared-rand-previous-value", K_PREVIOUS_SRV,EQ(2),       NO_OBJ ),
 +  T01("shared-rand-current-value",  K_CURRENT_SRV, EQ(2),       NO_OBJ ),
 +  T0N("package",               K_PACKAGE,          CONCAT_ARGS, NO_OBJ ),
 +  T01("recommended-client-protocols", K_RECOMMENDED_CLIENT_PROTOCOLS,
 +      CONCAT_ARGS, NO_OBJ ),
 +  T01("recommended-relay-protocols", K_RECOMMENDED_RELAY_PROTOCOLS,
 +      CONCAT_ARGS, NO_OBJ ),
 +  T01("required-client-protocols",    K_REQUIRED_CLIENT_PROTOCOLS,
 +      CONCAT_ARGS, NO_OBJ ),
 +  T01("required-relay-protocols",    K_REQUIRED_RELAY_PROTOCOLS,
 +      CONCAT_ARGS, NO_OBJ ),
 +
 +#include "feature/dirparse/authcert_members.i"
 +
 +  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
 +  T1( "contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
 +  T1( "dir-source",          K_DIR_SOURCE,      GE(6),       NO_OBJ ),
 +  T01("legacy-dir-key",      K_LEGACY_DIR_KEY,  GE(1),       NO_OBJ ),
 +  T1( "known-flags",         K_KNOWN_FLAGS,     CONCAT_ARGS, NO_OBJ ),
 +  T01("client-versions",     K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ),
 +  T01("server-versions",     K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ),
 +  T1( "consensus-methods",   K_CONSENSUS_METHODS, GE(1),     NO_OBJ ),
 +
 +  END_OF_TABLE
 +};
 +
 +/** List of tokens recognized in V3 networkstatus consensuses. */
 +static token_rule_t networkstatus_consensus_token_table[] = {
 +  T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
 +                                                   GE(1),       NO_OBJ ),
 +  T1("vote-status",            K_VOTE_STATUS,      GE(1),       NO_OBJ ),
 +  T1("valid-after",            K_VALID_AFTER,      CONCAT_ARGS, NO_OBJ ),
 +  T1("fresh-until",            K_FRESH_UNTIL,      CONCAT_ARGS, NO_OBJ ),
 +  T1("valid-until",            K_VALID_UNTIL,      CONCAT_ARGS, NO_OBJ ),
 +  T1("voting-delay",           K_VOTING_DELAY,     GE(2),       NO_OBJ ),
 +
 +  T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
 +
 +  T1N("dir-source",          K_DIR_SOURCE,          GE(6),   NO_OBJ ),
 +  T1N("contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
 +  T1N("vote-digest",         K_VOTE_DIGEST,         GE(1),   NO_OBJ ),
 +
 +  T1( "known-flags",         K_KNOWN_FLAGS,     CONCAT_ARGS, NO_OBJ ),
 +
 +  T01("client-versions",     K_CLIENT_VERSIONS, CONCAT_ARGS, NO_OBJ ),
 +  T01("server-versions",     K_SERVER_VERSIONS, CONCAT_ARGS, NO_OBJ ),
 +  T01("consensus-method",    K_CONSENSUS_METHOD,    EQ(1),   NO_OBJ),
 +  T01("params",                K_PARAMS,           ARGS,        NO_OBJ ),
 +
 +  T01("shared-rand-previous-value", K_PREVIOUS_SRV, EQ(2),   NO_OBJ ),
 +  T01("shared-rand-current-value",  K_CURRENT_SRV,  EQ(2),   NO_OBJ ),
 +
 +  T01("recommended-client-protocols", K_RECOMMENDED_CLIENT_PROTOCOLS,
 +      CONCAT_ARGS, NO_OBJ ),
 +  T01("recommended-relay-protocols", K_RECOMMENDED_RELAY_PROTOCOLS,
 +      CONCAT_ARGS, NO_OBJ ),
 +  T01("required-client-protocols",    K_REQUIRED_CLIENT_PROTOCOLS,
 +      CONCAT_ARGS, NO_OBJ ),
 +  T01("required-relay-protocols",    K_REQUIRED_RELAY_PROTOCOLS,
 +      CONCAT_ARGS, NO_OBJ ),
 +
 +  END_OF_TABLE
 +};
 +
 +/** List of tokens recognized in the footer of v1 directory footers. */
 +static token_rule_t networkstatus_vote_footer_token_table[] = {
 +  T01("directory-footer",    K_DIRECTORY_FOOTER,    NO_ARGS,   NO_OBJ ),
 +  T01("bandwidth-weights",   K_BW_WEIGHTS,          ARGS,      NO_OBJ ),
 +  T(  "directory-signature", K_DIRECTORY_SIGNATURE, GE(2),     NEED_OBJ ),
 +  END_OF_TABLE
 +};
 +
 +/** Try to find the start and end of the signed portion of a networkstatus
 + * document in <b>s</b>. On success, set <b>start_out</b> to the first
 + * character of the document, and <b>end_out</b> to a position one after the
 + * final character of the signed document, and return 0.  On failure, return
 + * -1. */
 +int
 +router_get_networkstatus_v3_signed_boundaries(const char *s,
++                                              size_t len,
 +                                              const char **start_out,
 +                                              const char **end_out)
 +{
-   return router_get_hash_impl_helper(s, strlen(s),
++  return router_get_hash_impl_helper(s, len,
 +                                     "network-status-version",
 +                                     "\ndirectory-signature",
 +                                     ' ', LOG_INFO,
 +                                     start_out, end_out);
 +}
 +
 +/** Set <b>digest_out</b> to the SHA3-256 digest of the signed portion of the
 + * networkstatus vote in <b>s</b> -- or of the entirety of <b>s</b> if no
 + * signed portion can be identified.  Return 0 on success, -1 on failure. */
 +int
 +router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
-                                            const char *s)
++                                           const char *s, size_t len)
 +{
 +  const char *start, *end;
-   if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) {
++  if (router_get_networkstatus_v3_signed_boundaries(s, len,
++                                                    &start, &end) < 0) {
 +    start = s;
-     end = s + strlen(s);
++    end = s + len;
 +  }
 +  tor_assert(start);
 +  tor_assert(end);
 +  return crypto_digest256((char*)digest_out, start, end-start,
 +                          DIGEST_SHA3_256);
 +}
 +
 +/** Set <b>digests</b> to all the digests of the consensus document in
 + * <b>s</b> */
 +int
- router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
++router_get_networkstatus_v3_hashes(const char *s, size_t len,
++                                   common_digests_t *digests)
 +{
-   return router_get_hashes_impl(s,strlen(s),digests,
++  return router_get_hashes_impl(s, len, digests,
 +                                "network-status-version",
 +                                "\ndirectory-signature",
 +                                ' ');
 +}
 +
 +/** Helper: given a string <b>s</b>, return the start of the next router-status
 + * object (starting with "r " at the start of a line).  If none is found,
 + * return the start of the directory footer, or the next directory signature.
 + * If none is found, return the end of the string. */
 +static inline const char *
- find_start_of_next_routerstatus(const char *s)
++find_start_of_next_routerstatus(const char *s, const char *s_eos)
 +{
 +  const char *eos, *footer, *sig;
-   if ((eos = strstr(s, "\nr ")))
++  if ((eos = tor_memstr(s, s_eos - s, "\nr ")))
 +    ++eos;
 +  else
-     eos = s + strlen(s);
++    eos = s_eos;
 +
 +  footer = tor_memstr(s, eos-s, "\ndirectory-footer");
 +  sig = tor_memstr(s, eos-s, "\ndirectory-signature");
 +
 +  if (footer && sig)
 +    return MIN(footer, sig) + 1;
 +  else if (footer)
 +    return footer+1;
 +  else if (sig)
 +    return sig+1;
 +  else
 +    return eos;
 +}
 +
 +/** Parse the GuardFraction string from a consensus or vote.
 + *
 + *  If <b>vote</b> or <b>vote_rs</b> are set the document getting
 + *  parsed is a vote routerstatus. Otherwise it's a consensus. This is
 + *  the same semantic as in routerstatus_parse_entry_from_string(). */
 +STATIC int
 +routerstatus_parse_guardfraction(const char *guardfraction_str,
 +                                 networkstatus_t *vote,
 +                                 vote_routerstatus_t *vote_rs,
 +                                 routerstatus_t *rs)
 +{
 +  int ok;
 +  const char *end_of_header = NULL;
 +  int is_consensus = !vote_rs;
 +  uint32_t guardfraction;
 +
 +  tor_assert(bool_eq(vote, vote_rs));
 +
 +  /* If this info comes from a consensus, but we should't apply
 +     guardfraction, just exit. */
 +  if (is_consensus && !should_apply_guardfraction(NULL)) {
 +    return 0;
 +  }
 +
 +  end_of_header = strchr(guardfraction_str, '=');
 +  if (!end_of_header) {
 +    return -1;
 +  }
 +
 +  guardfraction = (uint32_t)tor_parse_ulong(end_of_header+1,
 +                                            10, 0, 100, &ok, NULL);
 +  if (!ok) {
 +    log_warn(LD_DIR, "Invalid GuardFraction %s", escaped(guardfraction_str));
 +    return -1;
 +  }
 +
 +  log_debug(LD_GENERAL, "[*] Parsed %s guardfraction '%s' for '%s'.",
 +            is_consensus ? "consensus" : "vote",
 +            guardfraction_str, rs->nickname);
 +
 +  if (!is_consensus) { /* We are parsing a vote */
 +    vote_rs->status.guardfraction_percentage = guardfraction;
 +    vote_rs->status.has_guardfraction = 1;
 +  } else {
 +    /* We are parsing a consensus. Only apply guardfraction to guards. */
 +    if (rs->is_possible_guard) {
 +      rs->guardfraction_percentage = guardfraction;
 +      rs->has_guardfraction = 1;
 +    } else {
 +      log_warn(LD_BUG, "Got GuardFraction for non-guard %s. "
 +               "This is not supposed to happen. Not applying. ", rs->nickname);
 +    }
 +  }
 +
 +  return 0;
 +}
 +
 +/** Given a string at *<b>s</b>, containing a routerstatus object, and an
 + * empty smartlist at <b>tokens</b>, parse and return the first router status
 + * object in the string, and advance *<b>s</b> to just after the end of the
 + * router status.  Return NULL and advance *<b>s</b> on error.
 + *
 + * If <b>vote</b> and <b>vote_rs</b> are provided, don't allocate a fresh
 + * routerstatus but use <b>vote_rs</b> instead.
 + *
 + * If <b>consensus_method</b> is nonzero, this routerstatus is part of a
 + * consensus, and we should parse it according to the method used to
 + * make that consensus.
 + *
 + * Parse according to the syntax used by the consensus flavor <b>flav</b>.
 + **/
 +STATIC routerstatus_t *
 +routerstatus_parse_entry_from_string(memarea_t *area,
-                                      const char **s, smartlist_t *tokens,
++                                     const char **s, const char *s_eos,
++                                     smartlist_t *tokens,
 +                                     networkstatus_t *vote,
 +                                     vote_routerstatus_t *vote_rs,
 +                                     int consensus_method,
 +                                     consensus_flavor_t flav)
 +{
 +  const char *eos, *s_dup = *s;
 +  routerstatus_t *rs = NULL;
 +  directory_token_t *tok;
 +  char timebuf[ISO_TIME_LEN+1];
 +  struct in_addr in;
 +  int offset = 0;
 +  tor_assert(tokens);
 +  tor_assert(bool_eq(vote, vote_rs));
 +
 +  if (!consensus_method)
 +    flav = FLAV_NS;
 +  tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC);
 +
-   eos = find_start_of_next_routerstatus(*s);
++  eos = find_start_of_next_routerstatus(*s, s_eos);
 +
 +  if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) {
 +    log_warn(LD_DIR, "Error tokenizing router status");
 +    goto err;
 +  }
 +  if (smartlist_len(tokens) < 1) {
 +    log_warn(LD_DIR, "Impossibly short router status");
 +    goto err;
 +  }
 +  tok = find_by_keyword(tokens, K_R);
 +  tor_assert(tok->n_args >= 7); /* guaranteed by GE(7) in K_R setup */
 +  if (flav == FLAV_NS) {
 +    if (tok->n_args < 8) {
 +      log_warn(LD_DIR, "Too few arguments to r");
 +      goto err;
 +    }
 +  } else if (flav == FLAV_MICRODESC) {
 +    offset = -1; /* There is no descriptor digest in an md consensus r line */
 +  }
 +
 +  if (vote_rs) {
 +    rs = &vote_rs->status;
 +  } else {
 +    rs = tor_malloc_zero(sizeof(routerstatus_t));
 +  }
 +
 +  if (!is_legal_nickname(tok->args[0])) {
 +    log_warn(LD_DIR,
 +             "Invalid nickname %s in router status; skipping.",
 +             escaped(tok->args[0]));
 +    goto err;
 +  }
 +  strlcpy(rs->nickname, tok->args[0], sizeof(rs->nickname));
 +
 +  if (digest_from_base64(rs->identity_digest, tok->args[1])) {
 +    log_warn(LD_DIR, "Error decoding identity digest %s",
 +             escaped(tok->args[1]));
 +    goto err;
 +  }
 +
 +  if (flav == FLAV_NS) {
 +    if (digest_from_base64(rs->descriptor_digest, tok->args[2])) {
 +      log_warn(LD_DIR, "Error decoding descriptor digest %s",
 +               escaped(tok->args[2]));
 +      goto err;
 +    }
 +  }
 +
 +  if (tor_snprintf(timebuf, sizeof(timebuf), "%s %s",
 +                   tok->args[3+offset], tok->args[4+offset]) < 0 ||
 +      parse_iso_time(timebuf, &rs->published_on)<0) {
 +    log_warn(LD_DIR, "Error parsing time '%s %s' [%d %d]",
 +             tok->args[3+offset], tok->args[4+offset],
 +             offset, (int)flav);
 +    goto err;
 +  }
 +
 +  if (tor_inet_aton(tok->args[5+offset], &in) == 0) {
 +    log_warn(LD_DIR, "Error parsing router address in network-status %s",
 +             escaped(tok->args[5+offset]));
 +    goto err;
 +  }
 +  rs->addr = ntohl(in.s_addr);
 +
 +  rs->or_port = (uint16_t) tor_parse_long(tok->args[6+offset],
 +                                         10,0,65535,NULL,NULL);
 +  rs->dir_port = (uint16_t) tor_parse_long(tok->args[7+offset],
 +                                           10,0,65535,NULL,NULL);
 +
 +  {
 +    smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
 +    if (a_lines) {
 +      find_single_ipv6_orport(a_lines, &rs->ipv6_addr, &rs->ipv6_orport);
 +      smartlist_free(a_lines);
 +    }
 +  }
 +
 +  tok = find_opt_by_keyword(tokens, K_S);
 +  if (tok && vote) {
 +    int i;
 +    vote_rs->flags = 0;
 +    for (i=0; i < tok->n_args; ++i) {
 +      int p = smartlist_string_pos(vote->known_flags, tok->args[i]);
 +      if (p >= 0) {
 +        vote_rs->flags |= (UINT64_C(1)<<p);
 +      } else {
 +        log_warn(LD_DIR, "Flags line had a flag %s not listed in known_flags.",
 +                 escaped(tok->args[i]));
 +        goto err;
 +      }
 +    }
 +  } else if (tok) {
 +    /* This is a consensus, not a vote. */
 +    int i;
 +    for (i=0; i < tok->n_args; ++i) {
 +      if (!strcmp(tok->args[i], "Exit"))
 +        rs->is_exit = 1;
 +      else if (!strcmp(tok->args[i], "Stable"))
 +        rs->is_stable = 1;
 +      else if (!strcmp(tok->args[i], "Fast"))
 +        rs->is_fast = 1;
 +      else if (!strcmp(tok->args[i], "Running"))
 +        rs->is_flagged_running = 1;
 +      else if (!strcmp(tok->args[i], "Named"))
 +        rs->is_named = 1;
 +      else if (!strcmp(tok->args[i], "Valid"))
 +        rs->is_valid = 1;
 +      else if (!strcmp(tok->args[i], "Guard"))
 +        rs->is_possible_guard = 1;
 +      else if (!strcmp(tok->args[i], "BadExit"))
 +        rs->is_bad_exit = 1;
 +      else if (!strcmp(tok->args[i], "Authority"))
 +        rs->is_authority = 1;
 +      else if (!strcmp(tok->args[i], "Unnamed") &&
 +               consensus_method >= 2) {
 +        /* Unnamed is computed right by consensus method 2 and later. */
 +        rs->is_unnamed = 1;
 +      } else if (!strcmp(tok->args[i], "HSDir")) {
 +        rs->is_hs_dir = 1;
 +      } else if (!strcmp(tok->args[i], "V2Dir")) {
 +        rs->is_v2_dir = 1;
 +      }
 +    }
 +    /* These are implied true by having been included in a consensus made
 +     * with a given method */
 +    rs->is_flagged_running = 1; /* Starting with consensus method 4. */
 +    rs->is_valid = 1; /* Starting with consensus method 24. */
 +  }
 +  {
 +    const char *protocols = NULL, *version = NULL;
 +    if ((tok = find_opt_by_keyword(tokens, K_PROTO))) {
 +      tor_assert(tok->n_args == 1);
 +      protocols = tok->args[0];
 +    }
 +    if ((tok = find_opt_by_keyword(tokens, K_V))) {
 +      tor_assert(tok->n_args == 1);
 +      version = tok->args[0];
 +      if (vote_rs) {
 +        vote_rs->version = tor_strdup(tok->args[0]);
 +      }
 +    }
 +
 +    summarize_protover_flags(&rs->pv, protocols, version);
 +  }
 +
 +  /* handle weighting/bandwidth info */
 +  if ((tok = find_opt_by_keyword(tokens, K_W))) {
 +    int i;
 +    for (i=0; i < tok->n_args; ++i) {
 +      if (!strcmpstart(tok->args[i], "Bandwidth=")) {
 +        int ok;
 +        rs->bandwidth_kb =
 +          (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1,
 +                                    10, 0, UINT32_MAX,
 +                                    &ok, NULL);
 +        if (!ok) {
 +          log_warn(LD_DIR, "Invalid Bandwidth %s", escaped(tok->args[i]));
 +          goto err;
 +        }
 +        rs->has_bandwidth = 1;
 +      } else if (!strcmpstart(tok->args[i], "Measured=") && vote_rs) {
 +        int ok;
 +        vote_rs->measured_bw_kb =
 +            (uint32_t)tor_parse_ulong(strchr(tok->args[i], '=')+1,
 +                                      10, 0, UINT32_MAX, &ok, NULL);
 +        if (!ok) {
 +          log_warn(LD_DIR, "Invalid Measured Bandwidth %s",
 +                   escaped(tok->args[i]));
 +          goto err;
 +        }
 +        vote_rs->has_measured_bw = 1;
 +        vote->has_measured_bws = 1;
 +      } else if (!strcmpstart(tok->args[i], "Unmeasured=1")) {
 +        rs->bw_is_unmeasured = 1;
 +      } else if (!strcmpstart(tok->args[i], "GuardFraction=")) {
 +        if (routerstatus_parse_guardfraction(tok->args[i],
 +                                             vote, vote_rs, rs) < 0) {
 +          goto err;
 +        }
 +      }
 +    }
 +  }
 +
 +  /* parse exit policy summaries */
 +  if ((tok = find_opt_by_keyword(tokens, K_P))) {
 +    tor_assert(tok->n_args == 1);
 +    if (strcmpstart(tok->args[0], "accept ") &&
 +        strcmpstart(tok->args[0], "reject ")) {
 +      log_warn(LD_DIR, "Unknown exit policy summary type %s.",
 +               escaped(tok->args[0]));
 +      goto err;
 +    }
 +    /* XXX weasel: parse this into ports and represent them somehow smart,
 +     * maybe not here but somewhere on if we need it for the client.
 +     * we should still parse it here to check it's valid tho.
 +     */
 +    rs->exitsummary = tor_strdup(tok->args[0]);
 +    rs->has_exitsummary = 1;
 +  }
 +
 +  if (vote_rs) {
 +    SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, t) {
 +      if (t->tp == K_M && t->n_args) {
 +        vote_microdesc_hash_t *line =
 +          tor_malloc(sizeof(vote_microdesc_hash_t));
 +        line->next = vote_rs->microdesc;
 +        line->microdesc_hash_line = tor_strdup(t->args[0]);
 +        vote_rs->microdesc = line;
 +      }
 +      if (t->tp == K_ID) {
 +        tor_assert(t->n_args >= 2);
 +        if (!strcmp(t->args[0], "ed25519")) {
 +          vote_rs->has_ed25519_listing = 1;
 +          if (strcmp(t->args[1], "none") &&
 +              digest256_from_base64((char*)vote_rs->ed25519_id,
 +                                    t->args[1])<0) {
 +            log_warn(LD_DIR, "Bogus ed25519 key in networkstatus vote");
 +            goto err;
 +          }
 +        }
 +      }
 +      if (t->tp == K_PROTO) {
 +        tor_assert(t->n_args == 1);
 +        vote_rs->protocols = tor_strdup(t->args[0]);
 +      }
 +    } SMARTLIST_FOREACH_END(t);
 +  } else if (flav == FLAV_MICRODESC) {
 +    tok = find_opt_by_keyword(tokens, K_M);
 +    if (tok) {
 +      tor_assert(tok->n_args);
 +      if (digest256_from_base64(rs->descriptor_digest, tok->args[0])) {
 +        log_warn(LD_DIR, "Error decoding microdescriptor digest %s",
 +                 escaped(tok->args[0]));
 +        goto err;
 +      }
 +    } else {
 +      log_info(LD_BUG, "Found an entry in networkstatus with no "
 +               "microdescriptor digest. (Router %s ($%s) at %s:%d.)",
 +               rs->nickname, hex_str(rs->identity_digest, DIGEST_LEN),
 +               fmt_addr32(rs->addr), rs->or_port);
 +    }
 +  }
 +
 +  if (!strcasecmp(rs->nickname, UNNAMED_ROUTER_NICKNAME))
 +    rs->is_named = 0;
 +
 +  goto done;
 + err:
 +  dump_desc(s_dup, "routerstatus entry");
 +  if (rs && !vote_rs)
 +    routerstatus_free(rs);
 +  rs = NULL;
 + done:
 +  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
 +  smartlist_clear(tokens);
 +  if (area) {
 +    DUMP_AREA(area, "routerstatus entry");
 +    memarea_clear(area);
 +  }
 +  *s = eos;
 +
 +  return rs;
 +}
 +
 +int
 +compare_vote_routerstatus_entries(const void **_a, const void **_b)
 +{
 +  const vote_routerstatus_t *a = *_a, *b = *_b;
 +  return fast_memcmp(a->status.identity_digest, b->status.identity_digest,
 +                     DIGEST_LEN);
 +}
 +
 +/** Verify the bandwidth weights of a network status document */
 +int
 +networkstatus_verify_bw_weights(networkstatus_t *ns, int consensus_method)
 +{
 +  int64_t G=0, M=0, E=0, D=0, T=0;
 +  double Wgg, Wgm, Wgd, Wmg, Wmm, Wme, Wmd, Weg, Wem, Wee, Wed;
 +  double Gtotal=0, Mtotal=0, Etotal=0;
 +  const char *casename = NULL;
 +  int valid = 1;
 +  (void) consensus_method;
 +
 +  const int64_t weight_scale = networkstatus_get_weight_scale_param(ns);
 +  tor_assert(weight_scale >= 1);
 +  Wgg = networkstatus_get_bw_weight(ns, "Wgg", -1);
 +  Wgm = networkstatus_get_bw_weight(ns, "Wgm", -1);
 +  Wgd = networkstatus_get_bw_weight(ns, "Wgd", -1);
 +  Wmg = networkstatus_get_bw_weight(ns, "Wmg", -1);
 +  Wmm = networkstatus_get_bw_weight(ns, "Wmm", -1);
 +  Wme = networkstatus_get_bw_weight(ns, "Wme", -1);
 +  Wmd = networkstatus_get_bw_weight(ns, "Wmd", -1);
 +  Weg = networkstatus_get_bw_weight(ns, "Weg", -1);
 +  Wem = networkstatus_get_bw_weight(ns, "Wem", -1);
 +  Wee = networkstatus_get_bw_weight(ns, "Wee", -1);
 +  Wed = networkstatus_get_bw_weight(ns, "Wed", -1);
 +
 +  if (Wgg<0 || Wgm<0 || Wgd<0 || Wmg<0 || Wmm<0 || Wme<0 || Wmd<0 || Weg<0
 +          || Wem<0 || Wee<0 || Wed<0) {
 +    log_warn(LD_BUG, "No bandwidth weights produced in consensus!");
 +    return 0;
 +  }
 +
 +  // First, sanity check basic summing properties that hold for all cases
 +  // We use > 1 as the check for these because they are computed as integers.
 +  // Sometimes there are rounding errors.
 +  if (fabs(Wmm - weight_scale) > 1) {
 +    log_warn(LD_BUG, "Wmm=%f != %"PRId64,
 +             Wmm, (weight_scale));
 +    valid = 0;
 +  }
 +
 +  if (fabs(Wem - Wee) > 1) {
 +    log_warn(LD_BUG, "Wem=%f != Wee=%f", Wem, Wee);
 +    valid = 0;
 +  }
 +
 +  if (fabs(Wgm - Wgg) > 1) {
 +    log_warn(LD_BUG, "Wgm=%f != Wgg=%f", Wgm, Wgg);
 +    valid = 0;
 +  }
 +
 +  if (fabs(Weg - Wed) > 1) {
 +    log_warn(LD_BUG, "Wed=%f != Weg=%f", Wed, Weg);
 +    valid = 0;
 +  }
 +
 +  if (fabs(Wgg + Wmg - weight_scale) > 0.001*weight_scale) {
 +    log_warn(LD_BUG, "Wgg=%f != %"PRId64" - Wmg=%f", Wgg,
 +             (weight_scale), Wmg);
 +    valid = 0;
 +  }
 +
 +  if (fabs(Wee + Wme - weight_scale) > 0.001*weight_scale) {
 +    log_warn(LD_BUG, "Wee=%f != %"PRId64" - Wme=%f", Wee,
 +             (weight_scale), Wme);
 +    valid = 0;
 +  }
 +
 +  if (fabs(Wgd + Wmd + Wed - weight_scale) > 0.001*weight_scale) {
 +    log_warn(LD_BUG, "Wgd=%f + Wmd=%f + Wed=%f != %"PRId64,
 +             Wgd, Wmd, Wed, (weight_scale));
 +    valid = 0;
 +  }
 +
 +  Wgg /= weight_scale;
 +  Wgm /= weight_scale; (void) Wgm; // unused from here on.
 +  Wgd /= weight_scale;
 +
 +  Wmg /= weight_scale;
 +  Wmm /= weight_scale;
 +  Wme /= weight_scale;
 +  Wmd /= weight_scale;
 +
 +  Weg /= weight_scale; (void) Weg; // unused from here on.
 +  Wem /= weight_scale; (void) Wem; // unused from here on.
 +  Wee /= weight_scale;
 +  Wed /= weight_scale;
 +
 +  // Then, gather G, M, E, D, T to determine case
 +  SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, routerstatus_t *, rs) {
 +    int is_exit = 0;
 +    /* Bug #2203: Don't count bad exits as exits for balancing */
 +    is_exit = rs->is_exit && !rs->is_bad_exit;
 +    if (rs->has_bandwidth) {
 +      T += rs->bandwidth_kb;
 +      if (is_exit && rs->is_possible_guard) {
 +        D += rs->bandwidth_kb;
 +        Gtotal += Wgd*rs->bandwidth_kb;
 +        Mtotal += Wmd*rs->bandwidth_kb;
 +        Etotal += Wed*rs->bandwidth_kb;
 +      } else if (is_exit) {
 +        E += rs->bandwidth_kb;
 +        Mtotal += Wme*rs->bandwidth_kb;
 +        Etotal += Wee*rs->bandwidth_kb;
 +      } else if (rs->is_possible_guard) {
 +        G += rs->bandwidth_kb;
 +        Gtotal += Wgg*rs->bandwidth_kb;
 +        Mtotal += Wmg*rs->bandwidth_kb;
 +      } else {
 +        M += rs->bandwidth_kb;
 +        Mtotal += Wmm*rs->bandwidth_kb;
 +      }
 +    } else {
 +      log_warn(LD_BUG, "Missing consensus bandwidth for router %s",
 +               routerstatus_describe(rs));
 +    }
 +  } SMARTLIST_FOREACH_END(rs);
 +
 +  // Finally, check equality conditions depending upon case 1, 2 or 3
 +  // Full equality cases: 1, 3b
 +  // Partial equality cases: 2b (E=G), 3a (M=E)
 +  // Fully unknown: 2a
 +  if (3*E >= T && 3*G >= T) {
 +    // Case 1: Neither are scarce
 +    casename = "Case 1";
 +    if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
 +      log_warn(LD_DIR,
 +               "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
 +               "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +               " T=%"PRId64". "
 +               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +               casename, Etotal, Mtotal,
 +               (G), (M), (E),
 +               (D), (T),
 +               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +      valid = 0;
 +    }
 +    if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
 +      log_warn(LD_DIR,
 +               "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
 +               "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +               " T=%"PRId64". "
 +               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +               casename, Etotal, Gtotal,
 +               (G), (M), (E),
 +               (D), (T),
 +               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +      valid = 0;
 +    }
 +    if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
 +      log_warn(LD_DIR,
 +               "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
 +               "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +               " T=%"PRId64". "
 +               "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +               casename, Mtotal, Gtotal,
 +               (G), (M), (E),
 +               (D), (T),
 +               Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +      valid = 0;
 +    }
 +  } else if (3*E < T && 3*G < T) {
 +    int64_t R = MIN(E, G);
 +    int64_t S = MAX(E, G);
 +    /*
 +     * Case 2: Both Guards and Exits are scarce
 +     * Balance D between E and G, depending upon
 +     * D capacity and scarcity. Devote no extra
 +     * bandwidth to middle nodes.
 +     */
 +    if (R+D < S) { // Subcase a
 +      double Rtotal, Stotal;
 +      if (E < G) {
 +        Rtotal = Etotal;
 +        Stotal = Gtotal;
 +      } else {
 +        Rtotal = Gtotal;
 +        Stotal = Etotal;
 +      }
 +      casename = "Case 2a";
 +      // Rtotal < Stotal
 +      if (Rtotal > Stotal) {
 +        log_warn(LD_DIR,
 +                   "Bw Weight Failure for %s: Rtotal %f > Stotal %f. "
 +                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +                   " T=%"PRId64". "
 +                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                   casename, Rtotal, Stotal,
 +                   (G), (M), (E),
 +                   (D), (T),
 +                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +        valid = 0;
 +      }
 +      // Rtotal < T/3
 +      if (3*Rtotal > T) {
 +        log_warn(LD_DIR,
 +                   "Bw Weight Failure for %s: 3*Rtotal %f > T "
 +                   "%"PRId64". G=%"PRId64" M=%"PRId64" E=%"PRId64
 +                   " D=%"PRId64" T=%"PRId64". "
 +                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                   casename, Rtotal*3, (T),
 +                   (G), (M), (E),
 +                   (D), (T),
 +                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +        valid = 0;
 +      }
 +      // Stotal < T/3
 +      if (3*Stotal > T) {
 +        log_warn(LD_DIR,
 +                   "Bw Weight Failure for %s: 3*Stotal %f > T "
 +                   "%"PRId64". G=%"PRId64" M=%"PRId64" E=%"PRId64
 +                   " D=%"PRId64" T=%"PRId64". "
 +                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                   casename, Stotal*3, (T),
 +                   (G), (M), (E),
 +                   (D), (T),
 +                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +        valid = 0;
 +      }
 +      // Mtotal > T/3
 +      if (3*Mtotal < T) {
 +        log_warn(LD_DIR,
 +                   "Bw Weight Failure for %s: 3*Mtotal %f < T "
 +                   "%"PRId64". "
 +                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +                   " T=%"PRId64". "
 +                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                   casename, Mtotal*3, (T),
 +                   (G), (M), (E),
 +                   (D), (T),
 +                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +        valid = 0;
 +      }
 +    } else { // Subcase b: R+D > S
 +      casename = "Case 2b";
 +
 +      /* Check the rare-M redirect case. */
 +      if (D != 0 && 3*M < T) {
 +        casename = "Case 2b (balanced)";
 +        if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
 +          log_warn(LD_DIR,
 +                   "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
 +                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +                   " T=%"PRId64". "
 +                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                   casename, Etotal, Mtotal,
 +                   (G), (M), (E),
 +                   (D), (T),
 +                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +          valid = 0;
 +        }
 +        if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
 +          log_warn(LD_DIR,
 +                   "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
 +                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +                   " T=%"PRId64". "
 +                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                   casename, Etotal, Gtotal,
 +                   (G), (M), (E),
 +                   (D), (T),
 +                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +          valid = 0;
 +        }
 +        if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
 +          log_warn(LD_DIR,
 +                   "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
 +                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +                   " T=%"PRId64". "
 +                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                   casename, Mtotal, Gtotal,
 +                   (G), (M), (E),
 +                   (D), (T),
 +                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +          valid = 0;
 +        }
 +      } else {
 +        if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
 +          log_warn(LD_DIR,
 +                   "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
 +                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +                   " T=%"PRId64". "
 +                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                   casename, Etotal, Gtotal,
 +                   (G), (M), (E),
 +                   (D), (T),
 +                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +          valid = 0;
 +        }
 +      }
 +    }
 +  } else { // if (E < T/3 || G < T/3) {
 +    int64_t S = MIN(E, G);
 +    int64_t NS = MAX(E, G);
 +    if (3*(S+D) < T) { // Subcase a:
 +      double Stotal;
 +      double NStotal;
 +      if (G < E) {
 +        casename = "Case 3a (G scarce)";
 +        Stotal = Gtotal;
 +        NStotal = Etotal;
 +      } else { // if (G >= E) {
 +        casename = "Case 3a (E scarce)";
 +        NStotal = Gtotal;
 +        Stotal = Etotal;
 +      }
 +      // Stotal < T/3
 +      if (3*Stotal > T) {
 +        log_warn(LD_DIR,
 +                   "Bw Weight Failure for %s: 3*Stotal %f > T "
 +                   "%"PRId64". G=%"PRId64" M=%"PRId64" E=%"PRId64
 +                   " D=%"PRId64" T=%"PRId64". "
 +                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                   casename, Stotal*3, (T),
 +                   (G), (M), (E),
 +                   (D), (T),
 +                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +        valid = 0;
 +      }
 +      if (NS >= M) {
 +        if (fabs(NStotal-Mtotal) > 0.01*MAX(NStotal,Mtotal)) {
 +          log_warn(LD_DIR,
 +                   "Bw Weight Failure for %s: NStotal %f != Mtotal %f. "
 +                   "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +                   " T=%"PRId64". "
 +                   "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                   casename, NStotal, Mtotal,
 +                   (G), (M), (E),
 +                   (D), (T),
 +                   Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +          valid = 0;
 +        }
 +      } else {
 +        // if NS < M, NStotal > T/3 because only one of G or E is scarce
 +        if (3*NStotal < T) {
 +          log_warn(LD_DIR,
 +                     "Bw Weight Failure for %s: 3*NStotal %f < T "
 +                     "%"PRId64". G=%"PRId64" M=%"PRId64
 +                     " E=%"PRId64" D=%"PRId64" T=%"PRId64". "
 +                     "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                     casename, NStotal*3, (T),
 +                     (G), (M), (E),
 +                     (D), (T),
 +                     Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +          valid = 0;
 +        }
 +      }
 +    } else { // Subcase b: S+D >= T/3
 +      casename = "Case 3b";
 +      if (fabs(Etotal-Mtotal) > 0.01*MAX(Etotal,Mtotal)) {
 +        log_warn(LD_DIR,
 +                 "Bw Weight Failure for %s: Etotal %f != Mtotal %f. "
 +                 "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +                 " T=%"PRId64". "
 +                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                 casename, Etotal, Mtotal,
 +                 (G), (M), (E),
 +                 (D), (T),
 +                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +        valid = 0;
 +      }
 +      if (fabs(Etotal-Gtotal) > 0.01*MAX(Etotal,Gtotal)) {
 +        log_warn(LD_DIR,
 +                 "Bw Weight Failure for %s: Etotal %f != Gtotal %f. "
 +                 "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +                 " T=%"PRId64". "
 +                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                 casename, Etotal, Gtotal,
 +                 (G), (M), (E),
 +                 (D), (T),
 +                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +        valid = 0;
 +      }
 +      if (fabs(Gtotal-Mtotal) > 0.01*MAX(Gtotal,Mtotal)) {
 +        log_warn(LD_DIR,
 +                 "Bw Weight Failure for %s: Mtotal %f != Gtotal %f. "
 +                 "G=%"PRId64" M=%"PRId64" E=%"PRId64" D=%"PRId64
 +                 " T=%"PRId64". "
 +                 "Wgg=%f Wgd=%f Wmg=%f Wme=%f Wmd=%f Wee=%f Wed=%f",
 +                 casename, Mtotal, Gtotal,
 +                 (G), (M), (E),
 +                 (D), (T),
 +                 Wgg, Wgd, Wmg, Wme, Wmd, Wee, Wed);
 +        valid = 0;
 +      }
 +    }
 +  }
 +
 +  if (valid)
 +    log_notice(LD_DIR, "Bandwidth-weight %s is verified and valid.",
 +               casename);
 +
 +  return valid;
 +}
 +
 +/** Check if a shared random value of type <b>srv_type</b> is in
 + *  <b>tokens</b>. If there is, parse it and set it to <b>srv_out</b>. Return
 + *  -1 on failure, 0 on success. The resulting srv is allocated on the heap and
 + *  it's the responsibility of the caller to free it. */
 +static int
 +extract_one_srv(smartlist_t *tokens, directory_keyword srv_type,
 +                sr_srv_t **srv_out)
 +{
 +  int ret = -1;
 +  directory_token_t *tok;
 +  sr_srv_t *srv = NULL;
 +  smartlist_t *chunks;
 +
 +  tor_assert(tokens);
 +
 +  chunks = smartlist_new();
 +  tok = find_opt_by_keyword(tokens, srv_type);
 +  if (!tok) {
 +    /* That's fine, no SRV is allowed. */
 +    ret = 0;
 +    goto end;
 +  }
 +  for (int i = 0; i < tok->n_args; i++) {
 +    smartlist_add(chunks, tok->args[i]);
 +  }
 +  srv = sr_parse_srv(chunks);
 +  if (srv == NULL) {
 +    log_warn(LD_DIR, "SR: Unparseable SRV %s", escaped(tok->object_body));
 +    goto end;
 +  }
 +  /* All is good. */
 +  *srv_out = srv;
 +  ret = 0;
 + end:
 +  smartlist_free(chunks);
 +  return ret;
 +}
 +
 +/** Extract any shared random values found in <b>tokens</b> and place them in
 + *  the networkstatus <b>ns</b>. */
 +static void
 +extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens)
 +{
 +  const char *voter_identity;
 +  networkstatus_voter_info_t *voter;
 +
 +  tor_assert(ns);
 +  tor_assert(tokens);
 +  /* Can be only one of them else code flow. */
 +  tor_assert(ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS);
 +
 +  if (ns->type == NS_TYPE_VOTE) {
 +    voter = smartlist_get(ns->voters, 0);
 +    tor_assert(voter);
 +    voter_identity = hex_str(voter->identity_digest,
 +                             sizeof(voter->identity_digest));
 +  } else {
 +    /* Consensus has multiple voters so no specific voter. */
 +    voter_identity = "consensus";
 +  }
 +
 +  /* We extract both, and on error everything is stopped because it means
 +   * the vote is malformed for the shared random value(s). */
 +  if (extract_one_srv(tokens, K_PREVIOUS_SRV, &ns->sr_info.previous_srv) < 0) {
 +    log_warn(LD_DIR, "SR: Unable to parse previous SRV from %s",
 +             voter_identity);
 +    /* Maybe we have a chance with the current SRV so let's try it anyway. */
 +  }
 +  if (extract_one_srv(tokens, K_CURRENT_SRV, &ns->sr_info.current_srv) < 0) {
 +    log_warn(LD_DIR, "SR: Unable to parse current SRV from %s",
 +             voter_identity);
 +  }
 +}
 +
 +/** Parse a v3 networkstatus vote, opinion, or consensus (depending on
 + * ns_type), from <b>s</b>, and return the result.  Return NULL on failure. */
 +networkstatus_t *
- networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
++networkstatus_parse_vote_from_string(const char *s,
++                                     size_t s_len,
++                                     const char **eos_out,
 +                                     networkstatus_type_t ns_type)
 +{
 +  smartlist_t *tokens = smartlist_new();
 +  smartlist_t *rs_tokens = NULL, *footer_tokens = NULL;
 +  networkstatus_voter_info_t *voter = NULL;
 +  networkstatus_t *ns = NULL;
 +  common_digests_t ns_digests;
 +  uint8_t sha3_as_signed[DIGEST256_LEN];
 +  const char *cert, *end_of_header, *end_of_footer, *s_dup = s;
 +  directory_token_t *tok;
 +  struct in_addr in;
 +  int i, inorder, n_signatures = 0;
 +  memarea_t *area = NULL, *rs_area = NULL;
 +  consensus_flavor_t flav = FLAV_NS;
 +  char *last_kwd=NULL;
++  const char *eos = s + s_len;
 +
 +  tor_assert(s);
 +
 +  if (eos_out)
 +    *eos_out = NULL;
 +
-   if (router_get_networkstatus_v3_hashes(s, &ns_digests) ||
-       router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) {
++  if (router_get_networkstatus_v3_hashes(s, s_len, &ns_digests) ||
++      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed,
++                                                 s, s_len)<0) {
 +    log_warn(LD_DIR, "Unable to compute digest of network-status");
 +    goto err;
 +  }
 +
 +  area = memarea_new();
-   end_of_header = find_start_of_next_routerstatus(s);
++  end_of_header = find_start_of_next_routerstatus(s, eos);
 +  if (tokenize_string(area, s, end_of_header, tokens,
 +                      (ns_type == NS_TYPE_CONSENSUS) ?
 +                      networkstatus_consensus_token_table :
 +                      networkstatus_token_table, 0)) {
 +    log_warn(LD_DIR, "Error tokenizing network-status header");
 +    goto err;
 +  }
 +
 +  ns = tor_malloc_zero(sizeof(networkstatus_t));
 +  memcpy(&ns->digests, &ns_digests, sizeof(ns_digests));
 +  memcpy(&ns->digest_sha3_as_signed, sha3_as_signed, sizeof(sha3_as_signed));
 +
 +  tok = find_by_keyword(tokens, K_NETWORK_STATUS_VERSION);
 +  tor_assert(tok);
 +  if (tok->n_args > 1) {
 +    int flavor = networkstatus_parse_flavor_name(tok->args[1]);
 +    if (flavor < 0) {
 +      log_warn(LD_DIR, "Can't parse document with unknown flavor %s",
 +               escaped(tok->args[1]));
 +      goto err;
 +    }
 +    ns->flavor = flav = flavor;
 +  }
 +  if (flav != FLAV_NS && ns_type != NS_TYPE_CONSENSUS) {
 +    log_warn(LD_DIR, "Flavor found on non-consensus networkstatus.");
 +    goto err;
 +  }
 +
 +  if (ns_type != NS_TYPE_CONSENSUS) {
 +    const char *end_of_cert = NULL;
-     if (!(cert = strstr(s, "\ndir-key-certificate-version")))
++    if (!(cert = tor_memstr(s, end_of_header - s,
++                            "\ndir-key-certificate-version")))
 +      goto err;
 +    ++cert;
-     ns->cert = authority_cert_parse_from_string(cert, &end_of_cert);
++    ns->cert = authority_cert_parse_from_string(cert, end_of_header - cert,
++                                                &end_of_cert);
 +    if (!ns->cert || !end_of_cert || end_of_cert > end_of_header)
 +      goto err;
 +  }
 +
 +  tok = find_by_keyword(tokens, K_VOTE_STATUS);
 +  tor_assert(tok->n_args);
 +  if (!strcmp(tok->args[0], "vote")) {
 +    ns->type = NS_TYPE_VOTE;
 +  } else if (!strcmp(tok->args[0], "consensus")) {
 +    ns->type = NS_TYPE_CONSENSUS;
 +  } else if (!strcmp(tok->args[0], "opinion")) {
 +    ns->type = NS_TYPE_OPINION;
 +  } else {
 +    log_warn(LD_DIR, "Unrecognized vote status %s in network-status",
 +             escaped(tok->args[0]));
 +    goto err;
 +  }
 +  if (ns_type != ns->type) {
 +    log_warn(LD_DIR, "Got the wrong kind of v3 networkstatus.");
 +    goto err;
 +  }
 +
 +  if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_OPINION) {
 +    tok = find_by_keyword(tokens, K_PUBLISHED);
 +    if (parse_iso_time(tok->args[0], &ns->published))
 +      goto err;
 +
 +    ns->supported_methods = smartlist_new();
 +    tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHODS);
 +    if (tok) {
 +      for (i=0; i < tok->n_args; ++i)
 +        smartlist_add_strdup(ns->supported_methods, tok->args[i]);
 +    } else {
 +      smartlist_add_strdup(ns->supported_methods, "1");
 +    }
 +  } else {
 +    tok = find_opt_by_keyword(tokens, K_CONSENSUS_METHOD);
 +    if (tok) {
 +      int num_ok;
 +      ns->consensus_method = (int)tor_parse_long(tok->args[0], 10, 1, INT_MAX,
 +                                                 &num_ok, NULL);
 +      if (!num_ok)
 +        goto err;
 +    } else {
 +      ns->consensus_method = 1;
 +    }
 +  }
 +
 +  if ((tok = find_opt_by_keyword(tokens, K_RECOMMENDED_CLIENT_PROTOCOLS)))
 +    ns->recommended_client_protocols = tor_strdup(tok->args[0]);
 +  if ((tok = find_opt_by_keyword(tokens, K_RECOMMENDED_RELAY_PROTOCOLS)))
 +    ns->recommended_relay_protocols = tor_strdup(tok->args[0]);
 +  if ((tok = find_opt_by_keyword(tokens, K_REQUIRED_CLIENT_PROTOCOLS)))
 +    ns->required_client_protocols = tor_strdup(tok->args[0]);
 +  if ((tok = find_opt_by_keyword(tokens, K_REQUIRED_RELAY_PROTOCOLS)))
 +    ns->required_relay_protocols = tor_strdup(tok->args[0]);
 +
 +  tok = find_by_keyword(tokens, K_VALID_AFTER);
 +  if (parse_iso_time(tok->args[0], &ns->valid_after))
 +    goto err;
 +
 +  tok = find_by_keyword(tokens, K_FRESH_UNTIL);
 +  if (parse_iso_time(tok->args[0], &ns->fresh_until))
 +    goto err;
 +
 +  tok = find_by_keyword(tokens, K_VALID_UNTIL);
 +  if (parse_iso_time(tok->args[0], &ns->valid_until))
 +    goto err;
 +
 +  tok = find_by_keyword(tokens, K_VOTING_DELAY);
 +  tor_assert(tok->n_args >= 2);
 +  {
 +    int ok;
 +    ns->vote_seconds =
 +      (int) tor_parse_long(tok->args[0], 10, 0, INT_MAX, &ok, NULL);
 +    if (!ok)
 +      goto err;
 +    ns->dist_seconds =
 +      (int) tor_parse_long(tok->args[1], 10, 0, INT_MAX, &ok, NULL);
 +    if (!ok)
 +      goto err;
 +  }
 +  if (ns->valid_after +
 +      (get_options()->TestingTorNetwork ?
 +       MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL) > ns->fresh_until) {
 +    log_warn(LD_DIR, "Vote/consensus freshness interval is too short");
 +    goto err;
 +  }
 +  if (ns->valid_after +
 +      (get_options()->TestingTorNetwork ?
 +       MIN_VOTE_INTERVAL_TESTING : MIN_VOTE_INTERVAL)*2 > ns->valid_until) {
 +    log_warn(LD_DIR, "Vote/consensus liveness interval is too short");
 +    goto err;
 +  }
 +  if (ns->vote_seconds < MIN_VOTE_SECONDS) {
 +    log_warn(LD_DIR, "Vote seconds is too short");
 +    goto err;
 +  }
 +  if (ns->dist_seconds < MIN_DIST_SECONDS) {
 +    log_warn(LD_DIR, "Dist seconds is too short");
 +    goto err;
 +  }
 +
 +  if ((tok = find_opt_by_keyword(tokens, K_CLIENT_VERSIONS))) {
 +    ns->client_versions = tor_strdup(tok->args[0]);
 +  }
 +  if ((tok = find_opt_by_keyword(tokens, K_SERVER_VERSIONS))) {
 +    ns->server_versions = tor_strdup(tok->args[0]);
 +  }
 +
 +  {
 +    smartlist_t *package_lst = find_all_by_keyword(tokens, K_PACKAGE);
 +    ns->package_lines = smartlist_new();
 +    if (package_lst) {
 +      SMARTLIST_FOREACH(package_lst, directory_token_t *, t,
 +                    smartlist_add_strdup(ns->package_lines, t->args[0]));
 +    }
 +    smartlist_free(package_lst);
 +  }
 +
 +  tok = find_by_keyword(tokens, K_KNOWN_FLAGS);
 +  ns->known_flags = smartlist_new();
 +  inorder = 1;
 +  for (i = 0; i < tok->n_args; ++i) {
 +    smartlist_add_strdup(ns->known_flags, tok->args[i]);
 +    if (i>0 && strcmp(tok->args[i-1], tok->args[i])>= 0) {
 +      log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]);
 +      inorder = 0;
 +    }
 +  }
 +  if (!inorder) {
 +    log_warn(LD_DIR, "known-flags not in order");
 +    goto err;
 +  }
 +  if (ns->type != NS_TYPE_CONSENSUS &&
 +      smartlist_len(ns->known_flags) > MAX_KNOWN_FLAGS_IN_VOTE) {
 +    /* If we allowed more than 64 flags in votes, then parsing them would make
 +     * us invoke undefined behavior whenever we used 1<<flagnum to do a
 +     * bit-shift. This is only for votes and opinions: consensus users don't
 +     * care about flags they don't recognize, and so don't build a bitfield
 +     * for them. */
 +    log_warn(LD_DIR, "Too many known-flags in consensus vote or opinion");
 +    goto err;
 +  }
 +
 +  tok = find_opt_by_keyword(tokens, K_PARAMS);
 +  if (tok) {
 +    int any_dups = 0;
 +    inorder = 1;
 +    ns->net_params = smartlist_new();
 +    for (i = 0; i < tok->n_args; ++i) {
 +      int ok=0;
 +      char *eq = strchr(tok->args[i], '=');
 +      size_t eq_pos;
 +      if (!eq) {
 +        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
 +        goto err;
 +      }
 +      eq_pos = eq-tok->args[i];
 +      tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL);
 +      if (!ok) {
 +        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
 +        goto err;
 +      }
 +      if (i > 0 && strcmp(tok->args[i-1], tok->args[i]) >= 0) {
 +        log_warn(LD_DIR, "%s >= %s", tok->args[i-1], tok->args[i]);
 +        inorder = 0;
 +      }
 +      if (last_kwd && eq_pos == strlen(last_kwd) &&
 +          fast_memeq(last_kwd, tok->args[i], eq_pos)) {
 +        log_warn(LD_DIR, "Duplicate value for %s parameter",
 +                 escaped(tok->args[i]));
 +        any_dups = 1;
 +      }
 +      tor_free(last_kwd);
 +      last_kwd = tor_strndup(tok->args[i], eq_pos);
 +      smartlist_add_strdup(ns->net_params, tok->args[i]);
 +    }
 +    if (!inorder) {
 +      log_warn(LD_DIR, "params not in order");
 +      goto err;
 +    }
 +    if (any_dups) {
 +      log_warn(LD_DIR, "Duplicate in parameters");
 +      goto err;
 +    }
 +  }
 +
 +  ns->voters = smartlist_new();
 +
 +  SMARTLIST_FOREACH_BEGIN(tokens, directory_token_t *, _tok) {
 +    tok = _tok;
 +    if (tok->tp == K_DIR_SOURCE) {
 +      tor_assert(tok->n_args >= 6);
 +
 +      if (voter)
 +        smartlist_add(ns->voters, voter);
 +      voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
 +      voter->sigs = smartlist_new();
 +      if (ns->type != NS_TYPE_CONSENSUS)
 +        memcpy(voter->vote_digest, ns_digests.d[DIGEST_SHA1], DIGEST_LEN);
 +
 +      voter->nickname = tor_strdup(tok->args[0]);
 +      if (strlen(tok->args[1]) != HEX_DIGEST_LEN ||
 +          base16_decode(voter->identity_digest, sizeof(voter->identity_digest),
 +                        tok->args[1], HEX_DIGEST_LEN)
 +                        != sizeof(voter->identity_digest)) {
 +        log_warn(LD_DIR, "Error decoding identity digest %s in "
 +                 "network-status document.", escaped(tok->args[1]));
 +        goto err;
 +      }
 +      if (ns->type != NS_TYPE_CONSENSUS &&
 +          tor_memneq(ns->cert->cache_info.identity_digest,
 +                 voter->identity_digest, DIGEST_LEN)) {
 +        log_warn(LD_DIR,"Mismatch between identities in certificate and vote");
 +        goto err;
 +      }
 +      if (ns->type != NS_TYPE_CONSENSUS) {
 +        if (authority_cert_is_blacklisted(ns->cert)) {
 +          log_warn(LD_DIR, "Rejecting vote signature made with blacklisted "
 +                   "signing key %s",
 +                   hex_str(ns->cert->signing_key_digest, DIGEST_LEN));
 +          goto err;
 +        }
 +      }
 +      voter->address = tor_strdup(tok->args[2]);
 +      if (!tor_inet_aton(tok->args[3], &in)) {
 +        log_warn(LD_DIR, "Error decoding IP address %s in network-status.",
 +                 escaped(tok->args[3]));
 +        goto err;
 +      }
 +      voter->addr = ntohl(in.s_addr);
 +      int ok;
 +      voter->dir_port = (uint16_t)
 +        tor_parse_long(tok->args[4], 10, 0, 65535, &ok, NULL);
 +      if (!ok)
 +        goto err;
 +      voter->or_port = (uint16_t)
 +        tor_parse_long(tok->args[5], 10, 0, 65535, &ok, NULL);
 +      if (!ok)
 +        goto err;
 +    } else if (tok->tp == K_CONTACT) {
 +      if (!voter || voter->contact) {
 +        log_warn(LD_DIR, "contact element is out of place.");
 +        goto err;
 +      }
 +      voter->contact = tor_strdup(tok->args[0]);
 +    } else if (tok->tp == K_VOTE_DIGEST) {
 +      tor_assert(ns->type == NS_TYPE_CONSENSUS);
 +      tor_assert(tok->n_args >= 1);
 +      if (!voter || ! tor_digest_is_zero(voter->vote_digest)) {
 +        log_warn(LD_DIR, "vote-digest element is out of place.");
 +        goto err;
 +      }
 +      if (strlen(tok->args[0]) != HEX_DIGEST_LEN ||
 +        base16_decode(voter->vote_digest, sizeof(voter->vote_digest),
 +                      tok->args[0], HEX_DIGEST_LEN)
 +                      != sizeof(voter->vote_digest)) {
 +        log_warn(LD_DIR, "Error decoding vote digest %s in "
 +                 "network-status consensus.", escaped(tok->args[0]));
 +        goto err;
 +      }
 +    }
 +  } SMARTLIST_FOREACH_END(_tok);
 +  if (voter) {
 +    smartlist_add(ns->voters, voter);
 +    voter = NULL;
 +  }
 +  if (smartlist_len(ns->voters) == 0) {
 +    log_warn(LD_DIR, "Missing dir-source elements in a networkstatus.");
 +    goto err;
 +  } else if (ns->type != NS_TYPE_CONSENSUS && smartlist_len(ns->voters) != 1) {
 +    log_warn(LD_DIR, "Too many dir-source elements in a vote networkstatus.");
 +    goto err;
 +  }
 +
 +  if (ns->type != NS_TYPE_CONSENSUS &&
 +      (tok = find_opt_by_keyword(tokens, K_LEGACY_DIR_KEY))) {
 +    int bad = 1;
 +    if (strlen(tok->args[0]) == HEX_DIGEST_LEN) {
 +      networkstatus_voter_info_t *voter_0 = smartlist_get(ns->voters, 0);
 +      if (base16_decode(voter_0->legacy_id_digest, DIGEST_LEN,
 +                        tok->args[0], HEX_DIGEST_LEN) != DIGEST_LEN)
 +        bad = 1;
 +      else
 +        bad = 0;
 +    }
 +    if (bad) {
 +      log_warn(LD_DIR, "Invalid legacy key digest %s on vote.",
 +               escaped(tok->args[0]));
 +    }
 +  }
 +
 +  /* If this is a vote document, check if information about the shared
 +     randomness protocol is included, and extract it. */
 +  if (ns->type == NS_TYPE_VOTE) {
 +    dirvote_parse_sr_commits(ns, tokens);
 +  }
 +  /* For both a vote and consensus, extract the shared random values. */
 +  if (ns->type == NS_TYPE_VOTE || ns->type == NS_TYPE_CONSENSUS) {
 +    extract_shared_random_srvs(ns, tokens);
 +  }
 +
 +  /* Parse routerstatus lines. */
 +  rs_tokens = smartlist_new();
 +  rs_area = memarea_new();
 +  s = end_of_header;
 +  ns->routerstatus_list = smartlist_new();
 +
-   while (!strcmpstart(s, "r ")) {
++  while (eos - s >= 2 && fast_memeq(s, "r ", 2)) {
 +    if (ns->type != NS_TYPE_CONSENSUS) {
 +      vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t));
-       if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns,
++      if (routerstatus_parse_entry_from_string(rs_area, &s, eos, rs_tokens, ns,
 +                                               rs, 0, 0)) {
 +        smartlist_add(ns->routerstatus_list, rs);
 +      } else {
 +        vote_routerstatus_free(rs);
 +      }
 +    } else {
 +      routerstatus_t *rs;
-       if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens,
++      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, eos,
++                                                     rs_tokens,
 +                                                     NULL, NULL,
 +                                                     ns->consensus_method,
 +                                                     flav))) {
 +        /* Use exponential-backoff scheduling when downloading microdescs */
 +        smartlist_add(ns->routerstatus_list, rs);
 +      }
 +    }
 +  }
 +  for (i = 1; i < smartlist_len(ns->routerstatus_list); ++i) {
 +    routerstatus_t *rs1, *rs2;
 +    if (ns->type != NS_TYPE_CONSENSUS) {
 +      vote_routerstatus_t *a = smartlist_get(ns->routerstatus_list, i-1);
 +      vote_routerstatus_t *b = smartlist_get(ns->routerstatus_list, i);
 +      rs1 = &a->status; rs2 = &b->status;
 +    } else {
 +      rs1 = smartlist_get(ns->routerstatus_list, i-1);
 +      rs2 = smartlist_get(ns->routerstatus_list, i);
 +    }
 +    if (fast_memcmp(rs1->identity_digest, rs2->identity_digest, DIGEST_LEN)
 +        >= 0) {
 +      log_warn(LD_DIR, "Networkstatus entries not sorted by identity digest");
 +      goto err;
 +    }
 +  }
 +  if (ns_type != NS_TYPE_CONSENSUS) {
 +    digest256map_t *ed_id_map = digest256map_new();
 +    SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, vote_routerstatus_t *,
 +                            vrs) {
 +      if (! vrs->has_ed25519_listing ||
 +          tor_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
 +        continue;
 +      if (digest256map_get(ed_id_map, vrs->ed25519_id) != NULL) {
 +        log_warn(LD_DIR, "Vote networkstatus ed25519 identities were not "
 +                 "unique");
 +        digest256map_free(ed_id_map, NULL);
 +        goto err;
 +      }
 +      digest256map_set(ed_id_map, vrs->ed25519_id, (void*)1);
 +    } SMARTLIST_FOREACH_END(vrs);
 +    digest256map_free(ed_id_map, NULL);
 +  }
 +
 +  /* Parse footer; check signature. */
 +  footer_tokens = smartlist_new();
-   if ((end_of_footer = strstr(s, "\nnetwork-status-version ")))
++  if ((end_of_footer = tor_memstr(s, eos-s, "\nnetwork-status-version ")))
 +    ++end_of_footer;
 +  else
-     end_of_footer = s + strlen(s);
++    end_of_footer = eos;
 +  if (tokenize_string(area,s, end_of_footer, footer_tokens,
 +                      networkstatus_vote_footer_token_table, 0)) {
 +    log_warn(LD_DIR, "Error tokenizing network-status vote footer.");
 +    goto err;
 +  }
 +
 +  {
 +    int found_sig = 0;
 +    SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) {
 +      tok = _tok;
 +      if (tok->tp == K_DIRECTORY_SIGNATURE)
 +        found_sig = 1;
 +      else if (found_sig) {
 +        log_warn(LD_DIR, "Extraneous token after first directory-signature");
 +        goto err;
 +      }
 +    } SMARTLIST_FOREACH_END(_tok);
 +  }
 +
 +  if ((tok = find_opt_by_keyword(footer_tokens, K_DIRECTORY_FOOTER))) {
 +    if (tok != smartlist_get(footer_tokens, 0)) {
 +      log_warn(LD_DIR, "Misplaced directory-footer token");
 +      goto err;
 +    }
 +  }
 +
 +  tok = find_opt_by_keyword(footer_tokens, K_BW_WEIGHTS);
 +  if (tok) {
 +    ns->weight_params = smartlist_new();
 +    for (i = 0; i < tok->n_args; ++i) {
 +      int ok=0;
 +      char *eq = strchr(tok->args[i], '=');
 +      if (!eq) {
 +        log_warn(LD_DIR, "Bad element '%s' in weight params",
 +                 escaped(tok->args[i]));
 +        goto err;
 +      }
 +      tor_parse_long(eq+1, 10, INT32_MIN, INT32_MAX, &ok, NULL);
 +      if (!ok) {
 +        log_warn(LD_DIR, "Bad element '%s' in params", escaped(tok->args[i]));
 +        goto err;
 +      }
 +      smartlist_add_strdup(ns->weight_params, tok->args[i]);
 +    }
 +  }
 +
 +  SMARTLIST_FOREACH_BEGIN(footer_tokens, directory_token_t *, _tok) {
 +    char declared_identity[DIGEST_LEN];
 +    networkstatus_voter_info_t *v;
 +    document_signature_t *sig;
 +    const char *id_hexdigest = NULL;
 +    const char *sk_hexdigest = NULL;
 +    digest_algorithm_t alg = DIGEST_SHA1;
 +    tok = _tok;
 +    if (tok->tp != K_DIRECTORY_SIGNATURE)
 +      continue;
 +    tor_assert(tok->n_args >= 2);
 +    if (tok->n_args == 2) {
 +      id_hexdigest = tok->args[0];
 +      sk_hexdigest = tok->args[1];
 +    } else {
 +      const char *algname = tok->args[0];
 +      int a;
 +      id_hexdigest = tok->args[1];
 +      sk_hexdigest = tok->args[2];
 +      a = crypto_digest_algorithm_parse_name(algname);
 +      if (a<0) {
 +        log_warn(LD_DIR, "Unknown digest algorithm %s; skipping",
 +                 escaped(algname));
 +        continue;
 +      }
 +      alg = a;
 +    }
 +
 +    if (!tok->object_type ||
 +        strcmp(tok->object_type, "SIGNATURE") ||
 +        tok->object_size < 128 || tok->object_size > 512) {
 +      log_warn(LD_DIR, "Bad object type or length on directory-signature");
 +      goto err;
 +    }
 +
 +    if (strlen(id_hexdigest) != HEX_DIGEST_LEN ||
 +        base16_decode(declared_identity, sizeof(declared_identity),
 +                      id_hexdigest, HEX_DIGEST_LEN)
 +                      != sizeof(declared_identity)) {
 +      log_warn(LD_DIR, "Error decoding declared identity %s in "
 +               "network-status document.", escaped(id_hexdigest));
 +      goto err;
 +    }
 +    if (!(v = networkstatus_get_voter_by_id(ns, declared_identity))) {
 +      log_warn(LD_DIR, "ID on signature on network-status document does "
 +               "not match any declared directory source.");
 +      goto err;
 +    }
 +    sig = tor_malloc_zero(sizeof(document_signature_t));
 +    memcpy(sig->identity_digest, v->identity_digest, DIGEST_LEN);
 +    sig->alg = alg;
 +    if (strlen(sk_hexdigest) != HEX_DIGEST_LEN ||
 +        base16_decode(sig->signing_key_digest, sizeof(sig->signing_key_digest),
 +                      sk_hexdigest, HEX_DIGEST_LEN)
 +                      != sizeof(sig->signing_key_digest)) {
 +      log_warn(LD_DIR, "Error decoding declared signing key digest %s in "
 +               "network-status document.", escaped(sk_hexdigest));
 +      tor_free(sig);
 +      goto err;
 +    }
 +
 +    if (ns->type != NS_TYPE_CONSENSUS) {
 +      if (tor_memneq(declared_identity, ns->cert->cache_info.identity_digest,
 +                 DIGEST_LEN)) {
 +        log_warn(LD_DIR, "Digest mismatch between declared and actual on "
 +                 "network-status vote.");
 +        tor_free(sig);
 +        goto err;
 +      }
 +    }
 +
 +    if (networkstatus_get_voter_sig_by_alg(v, sig->alg)) {
 +      /* We already parsed a vote with this algorithm from this voter. Use the
 +         first one. */
 +      log_fn(LOG_PROTOCOL_WARN, LD_DIR, "We received a networkstatus "
 +             "that contains two signatures from the same voter with the same "
 +             "algorithm. Ignoring the second signature.");
 +      tor_free(sig);
 +      continue;
 +    }
 +
 +    if (ns->type != NS_TYPE_CONSENSUS) {
 +      if (check_signature_token(ns_digests.d[DIGEST_SHA1], DIGEST_LEN,
 +                                tok, ns->cert->signing_key, 0,
 +                                "network-status document")) {
 +        tor_free(sig);
 +        goto err;
 +      }
 +      sig->good_signature = 1;
 +    } else {
 +      if (tok->object_size >= INT_MAX || tok->object_size >= SIZE_T_CEILING) {
 +        tor_free(sig);
 +        goto err;
 +      }
 +      sig->signature = tor_memdup(tok->object_body, tok->object_size);
 +      sig->signature_len = (int) tok->object_size;
 +    }
 +    smartlist_add(v->sigs, sig);
 +
 +    ++n_signatures;
 +  } SMARTLIST_FOREACH_END(_tok);
 +
 +  if (! n_signatures) {
 +    log_warn(LD_DIR, "No signatures on networkstatus document.");
 +    goto err;
 +  } else if (ns->type == NS_TYPE_VOTE && n_signatures != 1) {
 +    log_warn(LD_DIR, "Received more than one signature on a "
 +             "network-status vote.");
 +    goto err;
 +  }
 +
 +  if (eos_out)
 +    *eos_out = end_of_footer;
 +
 +  goto done;
 + err:
 +  dump_desc(s_dup, "v3 networkstatus");
 +  networkstatus_vote_free(ns);
 +  ns = NULL;
 + done:
 +  if (tokens) {
 +    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
 +    smartlist_free(tokens);
 +  }
 +  if (voter) {
 +    if (voter->sigs) {
 +      SMARTLIST_FOREACH(voter->sigs, document_signature_t *, sig,
 +                        document_signature_free(sig));
 +      smartlist_free(voter->sigs);
 +    }
 +    tor_free(voter->nickname);
 +    tor_free(voter->address);
 +    tor_free(voter->contact);
 +    tor_free(voter);
 +  }
 +  if (rs_tokens) {
 +    SMARTLIST_FOREACH(rs_tokens, directory_token_t *, t, token_clear(t));
 +    smartlist_free(rs_tokens);
 +  }
 +  if (footer_tokens) {
 +    SMARTLIST_FOREACH(footer_tokens, directory_token_t *, t, token_clear(t));
 +    smartlist_free(footer_tokens);
 +  }
 +  if (area) {
 +    DUMP_AREA(area, "v3 networkstatus");
 +    memarea_drop_all(area);
 +  }
 +  if (rs_area)
 +    memarea_drop_all(rs_area);
 +  tor_free(last_kwd);
 +
 +  return ns;
 +}
diff --cc src/feature/dirparse/ns_parse.h
index 22438d73a,000000000..85d9ded68
mode 100644,000000..100644
--- a/src/feature/dirparse/ns_parse.h
+++ b/src/feature/dirparse/ns_parse.h
@@@ -1,45 -1,0 +1,47 @@@
 +/* Copyright (c) 2001 Matej Pfajfar.
 + * Copyright (c) 2001-2004, Roger Dingledine.
 + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
 + * Copyright (c) 2007-2018, The Tor Project, Inc. */
 +/* See LICENSE for licensing information */
 +
 +/**
 + * \file ns_parse.h
 + * \brief Header file for ns_parse.c.
 + **/
 +
 +#ifndef TOR_NS_PARSE_H
 +#define TOR_NS_PARSE_H
 +
- int router_get_networkstatus_v3_hashes(const char *s,
++int router_get_networkstatus_v3_hashes(const char *s, size_t len,
 +                                       common_digests_t *digests);
- int router_get_networkstatus_v3_signed_boundaries(const char *s,
++int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len,
 +                                                  const char **start_out,
 +                                                  const char **end_out);
 +int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
-                                                const char *s);
++                                               const char *s, size_t len);
 +int compare_vote_routerstatus_entries(const void **_a, const void **_b);
 +
 +int networkstatus_verify_bw_weights(networkstatus_t *ns, int);
 +enum networkstatus_type_t;
 +networkstatus_t *networkstatus_parse_vote_from_string(const char *s,
++                                           size_t len,
 +                                           const char **eos_out,
 +                                           enum networkstatus_type_t ns_type);
 +
 +#ifdef NS_PARSE_PRIVATE
 +STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str,
 +                                            networkstatus_t *vote,
 +                                            vote_routerstatus_t *vote_rs,
 +                                            routerstatus_t *rs);
 +struct memarea_t;
 +STATIC routerstatus_t *routerstatus_parse_entry_from_string(
 +                                     struct memarea_t *area,
-                                      const char **s, smartlist_t *tokens,
++                                     const char **s, const char *eos,
++                                     smartlist_t *tokens,
 +                                     networkstatus_t *vote,
 +                                     vote_routerstatus_t *vote_rs,
 +                                     int consensus_method,
 +                                     consensus_flavor_t flav);
 +#endif
 +
 +#endif
diff --cc src/feature/nodelist/authcert.c
index b111422d0,000000000..2c4915e91
mode 100644,000000..100644
--- a/src/feature/nodelist/authcert.c
+++ b/src/feature/nodelist/authcert.c
@@@ -1,1208 -1,0 +1,1209 @@@
 +/* Copyright (c) 2001 Matej Pfajfar.
 + * Copyright (c) 2001-2004, Roger Dingledine.
 + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
 + * Copyright (c) 2007-2018, The Tor Project, Inc. */
 +/* See LICENSE for licensing information */
 +
 +/**
 + * \file authcert.c
 + * \brief Code to maintain directory authorities' certificates.
 + *
 + * Authority certificates are signed with authority identity keys; they
 + * are used to authenticate shorter-term authority signing keys. We
 + * fetch them when we find a consensus or a vote that has been signed
 + * with a signing key we don't recognize.  We cache them on disk and
 + * load them on startup.  Authority operators generate them with the
 + * "tor-gencert" utility.
 + */
 +
 +#include "core/or/or.h"
 +
 +#include "app/config/config.h"
 +#include "core/mainloop/connection.h"
 +#include "core/mainloop/mainloop.h"
 +#include "core/or/policies.h"
 +#include "feature/client/bridges.h"
 +#include "feature/dirauth/authmode.h"
 +#include "feature/dirclient/dirclient.h"
 +#include "feature/dirclient/dlstatus.h"
 +#include "feature/dircommon/directory.h"
 +#include "feature/dircommon/fp_pair.h"
 +#include "feature/dirparse/authcert_parse.h"
 +#include "feature/nodelist/authcert.h"
 +#include "feature/nodelist/dirlist.h"
 +#include "feature/nodelist/networkstatus.h"
 +#include "feature/nodelist/node_select.h"
 +#include "feature/nodelist/nodelist.h"
 +#include "feature/nodelist/routerlist.h"
 +#include "feature/relay/routermode.h"
 +
 +#include "core/or/connection_st.h"
 +#include "feature/dirclient/dir_server_st.h"
 +#include "feature/dircommon/dir_connection_st.h"
 +#include "feature/nodelist/authority_cert_st.h"
 +#include "feature/nodelist/document_signature_st.h"
 +#include "feature/nodelist/networkstatus_st.h"
 +#include "feature/nodelist/networkstatus_voter_info_st.h"
 +#include "feature/nodelist/node_st.h"
 +
 +DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
 +#define DSMAP_FOREACH(map, keyvar, valvar) \
 +  DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \
 +                    valvar)
 +#define dsmap_free(map, fn) MAP_FREE_AND_NULL(dsmap, (map), (fn))
 +
 +/* Forward declaration for cert_list_t */
 +typedef struct cert_list_t cert_list_t;
 +
 +static void download_status_reset_by_sk_in_cl(cert_list_t *cl,
 +                                              const char *digest);
 +static int download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
 +                                                const char *digest,
 +                                                time_t now);
 +static void list_pending_fpsk_downloads(fp_pair_map_t *result);
 +
 +/** List of certificates for a single authority, and download status for
 + * latest certificate.
 + */
 +struct cert_list_t {
 +  /*
 +   * The keys of download status map are cert->signing_key_digest for pending
 +   * downloads by (identity digest/signing key digest) pair; functions such
 +   * as authority_cert_get_by_digest() already assume these are unique.
 +   */
 +  struct digest_ds_map_t *dl_status_map;
 +  /* There is also a dlstatus for the download by identity key only */
 +  download_status_t dl_status_by_id;
 +  smartlist_t *certs;
 +};
 +/** Map from v3 identity key digest to cert_list_t. */
 +static digestmap_t *trusted_dir_certs = NULL;
 +
 +/** True iff any key certificate in at least one member of
 + * <b>trusted_dir_certs</b> has changed since we last flushed the
 + * certificates to disk. */
 +static int trusted_dir_servers_certs_changed = 0;
 +
 +/** Initialise schedule, want_authority, and increment_on in the download
 + * status dlstatus, then call download_status_reset() on it.
 + * It is safe to call this function or download_status_reset() multiple times
 + * on a new dlstatus. But it should *not* be called after a dlstatus has been
 + * used to count download attempts or failures. */
 +static void
 +download_status_cert_init(download_status_t *dlstatus)
 +{
 +  dlstatus->schedule = DL_SCHED_CONSENSUS;
 +  dlstatus->want_authority = DL_WANT_ANY_DIRSERVER;
 +  dlstatus->increment_on = DL_SCHED_INCREMENT_FAILURE;
 +  dlstatus->last_backoff_position = 0;
 +  dlstatus->last_delay_used = 0;
 +
 +  /* Use the new schedule to set next_attempt_at */
 +  download_status_reset(dlstatus);
 +}
 +
 +/** Reset the download status of a specified element in a dsmap */
 +static void
 +download_status_reset_by_sk_in_cl(cert_list_t *cl, const char *digest)
 +{
 +  download_status_t *dlstatus = NULL;
 +
 +  tor_assert(cl);
 +  tor_assert(digest);
 +
 +  /* Make sure we have a dsmap */
 +  if (!(cl->dl_status_map)) {
 +    cl->dl_status_map = dsmap_new();
 +  }
 +  /* Look for a download_status_t in the map with this digest */
 +  dlstatus = dsmap_get(cl->dl_status_map, digest);
 +  /* Got one? */
 +  if (!dlstatus) {
 +    /* Insert before we reset */
 +    dlstatus = tor_malloc_zero(sizeof(*dlstatus));
 +    dsmap_set(cl->dl_status_map, digest, dlstatus);
 +    download_status_cert_init(dlstatus);
 +  }
 +  tor_assert(dlstatus);
 +  /* Go ahead and reset it */
 +  download_status_reset(dlstatus);
 +}
 +
 +/**
 + * Return true if the download for this signing key digest in cl is ready
 + * to be re-attempted.
 + */
 +static int
 +download_status_is_ready_by_sk_in_cl(cert_list_t *cl,
 +                                     const char *digest,
 +                                     time_t now)
 +{
 +  int rv = 0;
 +  download_status_t *dlstatus = NULL;
 +
 +  tor_assert(cl);
 +  tor_assert(digest);
 +
 +  /* Make sure we have a dsmap */
 +  if (!(cl->dl_status_map)) {
 +    cl->dl_status_map = dsmap_new();
 +  }
 +  /* Look for a download_status_t in the map with this digest */
 +  dlstatus = dsmap_get(cl->dl_status_map, digest);
 +  /* Got one? */
 +  if (dlstatus) {
 +    /* Use download_status_is_ready() */
 +    rv = download_status_is_ready(dlstatus, now);
 +  } else {
 +    /*
 +     * If we don't know anything about it, return 1, since we haven't
 +     * tried this one before.  We need to create a new entry here,
 +     * too.
 +     */
 +    dlstatus = tor_malloc_zero(sizeof(*dlstatus));
 +    download_status_cert_init(dlstatus);
 +    dsmap_set(cl->dl_status_map, digest, dlstatus);
 +    rv = 1;
 +  }
 +
 +  return rv;
 +}
 +
 +/** Helper: Return the cert_list_t for an authority whose authority ID is
 + * <b>id_digest</b>, allocating a new list if necessary. */
 +static cert_list_t *
 +get_cert_list(const char *id_digest)
 +{
 +  cert_list_t *cl;
 +  if (!trusted_dir_certs)
 +    trusted_dir_certs = digestmap_new();
 +  cl = digestmap_get(trusted_dir_certs, id_digest);
 +  if (!cl) {
 +    cl = tor_malloc_zero(sizeof(cert_list_t));
 +    download_status_cert_init(&cl->dl_status_by_id);
 +    cl->certs = smartlist_new();
 +    cl->dl_status_map = dsmap_new();
 +    digestmap_set(trusted_dir_certs, id_digest, cl);
 +  }
 +  return cl;
 +}
 +
 +/** Return a list of authority ID digests with potentially enumerable lists
 + * of download_status_t objects; used by controller GETINFO queries.
 + */
 +
 +MOCK_IMPL(smartlist_t *,
 +list_authority_ids_with_downloads, (void))
 +{
 +  smartlist_t *ids = smartlist_new();
 +  digestmap_iter_t *i;
 +  const char *digest;
 +  char *tmp;
 +  void *cl;
 +
 +  if (trusted_dir_certs) {
 +    for (i = digestmap_iter_init(trusted_dir_certs);
 +         !(digestmap_iter_done(i));
 +         i = digestmap_iter_next(trusted_dir_certs, i)) {
 +      /*
 +       * We always have at least dl_status_by_id to query, so no need to
 +       * probe deeper than the existence of a cert_list_t.
 +       */
 +      digestmap_iter_get(i, &digest, &cl);
 +      tmp = tor_malloc(DIGEST_LEN);
 +      memcpy(tmp, digest, DIGEST_LEN);
 +      smartlist_add(ids, tmp);
 +    }
 +  }
 +  /* else definitely no downloads going since nothing even has a cert list */
 +
 +  return ids;
 +}
 +
 +/** Given an authority ID digest, return a pointer to the default download
 + * status, or NULL if there is no such entry in trusted_dir_certs */
 +
 +MOCK_IMPL(download_status_t *,
 +id_only_download_status_for_authority_id, (const char *digest))
 +{
 +  download_status_t *dl = NULL;
 +  cert_list_t *cl;
 +
 +  if (trusted_dir_certs) {
 +    cl = digestmap_get(trusted_dir_certs, digest);
 +    if (cl) {
 +      dl = &(cl->dl_status_by_id);
 +    }
 +  }
 +
 +  return dl;
 +}
 +
 +/** Given an authority ID digest, return a smartlist of signing key digests
 + * for which download_status_t is potentially queryable, or NULL if no such
 + * authority ID digest is known. */
 +
 +MOCK_IMPL(smartlist_t *,
 +list_sk_digests_for_authority_id, (const char *digest))
 +{
 +  smartlist_t *sks = NULL;
 +  cert_list_t *cl;
 +  dsmap_iter_t *i;
 +  const char *sk_digest;
 +  char *tmp;
 +  download_status_t *dl;
 +
 +  if (trusted_dir_certs) {
 +    cl = digestmap_get(trusted_dir_certs, digest);
 +    if (cl) {
 +      sks = smartlist_new();
 +      if (cl->dl_status_map) {
 +        for (i = dsmap_iter_init(cl->dl_status_map);
 +             !(dsmap_iter_done(i));
 +             i = dsmap_iter_next(cl->dl_status_map, i)) {
 +          /* Pull the digest out and add it to the list */
 +          dsmap_iter_get(i, &sk_digest, &dl);
 +          tmp = tor_malloc(DIGEST_LEN);
 +          memcpy(tmp, sk_digest, DIGEST_LEN);
 +          smartlist_add(sks, tmp);
 +        }
 +      }
 +    }
 +  }
 +
 +  return sks;
 +}
 +
 +/** Given an authority ID digest and a signing key digest, return the
 + * download_status_t or NULL if none exists. */
 +
 +MOCK_IMPL(download_status_t *,
 +download_status_for_authority_id_and_sk,(const char *id_digest,
 +                                         const char *sk_digest))
 +{
 +  download_status_t *dl = NULL;
 +  cert_list_t *cl = NULL;
 +
 +  if (trusted_dir_certs) {
 +    cl = digestmap_get(trusted_dir_certs, id_digest);
 +    if (cl && cl->dl_status_map) {
 +      dl = dsmap_get(cl->dl_status_map, sk_digest);
 +    }
 +  }
 +
 +  return dl;
 +}
 +
 +#define cert_list_free(val) \
 +  FREE_AND_NULL(cert_list_t, cert_list_free_, (val))
 +
 +/** Release all space held by a cert_list_t */
 +static void
 +cert_list_free_(cert_list_t *cl)
 +{
 +  if (!cl)
 +    return;
 +
 +  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
 +                    authority_cert_free(cert));
 +  smartlist_free(cl->certs);
 +  dsmap_free(cl->dl_status_map, tor_free_);
 +  tor_free(cl);
 +}
 +
 +/** Wrapper for cert_list_free so we can pass it to digestmap_free */
 +static void
 +cert_list_free_void(void *cl)
 +{
 +  cert_list_free_(cl);
 +}
 +
 +/** Reload the cached v3 key certificates from the cached-certs file in
 + * the data directory. Return 0 on success, -1 on failure. */
 +int
 +trusted_dirs_reload_certs(void)
 +{
 +  char *filename;
 +  char *contents;
 +  int r;
 +
 +  filename = get_cachedir_fname("cached-certs");
 +  contents = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
 +  tor_free(filename);
 +  if (!contents)
 +    return 0;
 +  r = trusted_dirs_load_certs_from_string(
 +        contents,
 +        TRUSTED_DIRS_CERTS_SRC_FROM_STORE, 1, NULL);
 +  tor_free(contents);
 +  return r;
 +}
 +
 +/** Helper: return true iff we already have loaded the exact cert
 + * <b>cert</b>. */
 +static inline int
 +already_have_cert(authority_cert_t *cert)
 +{
 +  cert_list_t *cl = get_cert_list(cert->cache_info.identity_digest);
 +
 +  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
 +  {
 +    if (tor_memeq(c->cache_info.signed_descriptor_digest,
 +                cert->cache_info.signed_descriptor_digest,
 +                DIGEST_LEN))
 +      return 1;
 +  });
 +  return 0;
 +}
 +
 +/** Load a bunch of new key certificates from the string <b>contents</b>.  If
 + * <b>source</b> is TRUSTED_DIRS_CERTS_SRC_FROM_STORE, the certificates are
 + * from the cache, and we don't need to flush them to disk.  If we are a
 + * dirauth loading our own cert, source is TRUSTED_DIRS_CERTS_SRC_SELF.
 + * Otherwise, source is download type: TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST
 + * or TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST.  If <b>flush</b> is true, we
 + * need to flush any changed certificates to disk now.  Return 0 on success,
 + * -1 if any certs fail to parse.
 + *
 + * If source_dir is non-NULL, it's the identity digest for a directory that
 + * we've just successfully retrieved certificates from, so try it first to
 + * fetch any missing certificates.
 + */
 +int
 +trusted_dirs_load_certs_from_string(const char *contents, int source,
 +                                    int flush, const char *source_dir)
 +{
 +  dir_server_t *ds;
 +  const char *s, *eos;
 +  int failure_code = 0;
 +  int from_store = (source == TRUSTED_DIRS_CERTS_SRC_FROM_STORE);
 +  int added_trusted_cert = 0;
 +
 +  for (s = contents; *s; s = eos) {
-     authority_cert_t *cert = authority_cert_parse_from_string(s, &eos);
++    authority_cert_t *cert = authority_cert_parse_from_string(s, strlen(s),
++                                                              &eos);
 +    cert_list_t *cl;
 +    if (!cert) {
 +      failure_code = -1;
 +      break;
 +    }
 +    ds = trusteddirserver_get_by_v3_auth_digest(
 +                                       cert->cache_info.identity_digest);
 +    log_debug(LD_DIR, "Parsed certificate for %s",
 +              ds ? ds->nickname : "unknown authority");
 +
 +    if (already_have_cert(cert)) {
 +      /* we already have this one. continue. */
 +      log_info(LD_DIR, "Skipping %s certificate for %s that we "
 +               "already have.",
 +               from_store ? "cached" : "downloaded",
 +               ds ? ds->nickname : "an old or new authority");
 +
 +      /*
 +       * A duplicate on download should be treated as a failure, so we call
 +       * authority_cert_dl_failed() to reset the download status to make sure
 +       * we can't try again.  Since we've implemented the fp-sk mechanism
 +       * to download certs by signing key, this should be much rarer than it
 +       * was and is perhaps cause for concern.
 +       */
 +      if (!from_store) {
 +        if (authdir_mode(get_options())) {
 +          log_warn(LD_DIR,
 +                   "Got a certificate for %s, but we already have it. "
 +                   "Maybe they haven't updated it. Waiting for a while.",
 +                   ds ? ds->nickname : "an old or new authority");
 +        } else {
 +          log_info(LD_DIR,
 +                   "Got a certificate for %s, but we already have it. "
 +                   "Maybe they haven't updated it. Waiting for a while.",
 +                   ds ? ds->nickname : "an old or new authority");
 +        }
 +
 +        /*
 +         * This is where we care about the source; authority_cert_dl_failed()
 +         * needs to know whether the download was by fp or (fp,sk) pair to
 +         * twiddle the right bit in the download map.
 +         */
 +        if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST) {
 +          authority_cert_dl_failed(cert->cache_info.identity_digest,
 +                                   NULL, 404);
 +        } else if (source == TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_SK_DIGEST) {
 +          authority_cert_dl_failed(cert->cache_info.identity_digest,
 +                                   cert->signing_key_digest, 404);
 +        }
 +      }
 +
 +      authority_cert_free(cert);
 +      continue;
 +    }
 +
 +    if (ds) {
 +      added_trusted_cert = 1;
 +      log_info(LD_DIR, "Adding %s certificate for directory authority %s with "
 +               "signing key %s", from_store ? "cached" : "downloaded",
 +               ds->nickname, hex_str(cert->signing_key_digest,DIGEST_LEN));
 +    } else {
 +      int adding = we_want_to_fetch_unknown_auth_certs(get_options());
 +      log_info(LD_DIR, "%s %s certificate for unrecognized directory "
 +               "authority with signing key %s",
 +               adding ? "Adding" : "Not adding",
 +               from_store ? "cached" : "downloaded",
 +               hex_str(cert->signing_key_digest,DIGEST_LEN));
 +      if (!adding) {
 +        authority_cert_free(cert);
 +        continue;
 +      }
 +    }
 +
 +    cl = get_cert_list(cert->cache_info.identity_digest);
 +    smartlist_add(cl->certs, cert);
 +    if (ds && cert->cache_info.published_on > ds->addr_current_at) {
 +      /* Check to see whether we should update our view of the authority's
 +       * address. */
 +      if (cert->addr && cert->dir_port &&
 +          (ds->addr != cert->addr ||
 +           ds->dir_port != cert->dir_port)) {
 +        char *a = tor_dup_ip(cert->addr);
 +        log_notice(LD_DIR, "Updating address for directory authority %s "
 +                   "from %s:%d to %s:%d based on certificate.",
 +                   ds->nickname, ds->address, (int)ds->dir_port,
 +                   a, cert->dir_port);
 +        tor_free(a);
 +        ds->addr = cert->addr;
 +        ds->dir_port = cert->dir_port;
 +      }
 +      ds->addr_current_at = cert->cache_info.published_on;
 +    }
 +
 +    if (!from_store)
 +      trusted_dir_servers_certs_changed = 1;
 +  }
 +
 +  if (flush)
 +    trusted_dirs_flush_certs_to_disk();
 +
 +  /* call this even if failure_code is <0, since some certs might have
 +   * succeeded, but only pass source_dir if there were no failures,
 +   * and at least one more authority certificate was added to the store.
 +   * This avoids retrying a directory that's serving bad or entirely duplicate
 +   * certificates. */
 +  if (failure_code == 0 && added_trusted_cert) {
 +    networkstatus_note_certs_arrived(source_dir);
 +  } else {
 +    networkstatus_note_certs_arrived(NULL);
 +  }
 +
 +  return failure_code;
 +}
 +
 +/** Save all v3 key certificates to the cached-certs file. */
 +void
 +trusted_dirs_flush_certs_to_disk(void)
 +{
 +  char *filename;
 +  smartlist_t *chunks;
 +
 +  if (!trusted_dir_servers_certs_changed || !trusted_dir_certs)
 +    return;
 +
 +  chunks = smartlist_new();
 +  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
 +    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
 +          {
 +            sized_chunk_t *c = tor_malloc(sizeof(sized_chunk_t));
 +            c->bytes = cert->cache_info.signed_descriptor_body;
 +            c->len = cert->cache_info.signed_descriptor_len;
 +            smartlist_add(chunks, c);
 +          });
 +  } DIGESTMAP_FOREACH_END;
 +
 +  filename = get_cachedir_fname("cached-certs");
 +  if (write_chunks_to_file(filename, chunks, 0, 0)) {
 +    log_warn(LD_FS, "Error writing certificates to disk.");
 +  }
 +  tor_free(filename);
 +  SMARTLIST_FOREACH(chunks, sized_chunk_t *, c, tor_free(c));
 +  smartlist_free(chunks);
 +
 +  trusted_dir_servers_certs_changed = 0;
 +}
 +
 +static int
 +compare_certs_by_pubdates(const void **_a, const void **_b)
 +{
 +  const authority_cert_t *cert1 = *_a, *cert2=*_b;
 +
 +  if (cert1->cache_info.published_on < cert2->cache_info.published_on)
 +    return -1;
 +  else if (cert1->cache_info.published_on >  cert2->cache_info.published_on)
 +    return 1;
 +  else
 +    return 0;
 +}
 +
 +/** Remove all expired v3 authority certificates that have been superseded for
 + * more than 48 hours or, if not expired, that were published more than 7 days
 + * before being superseded. (If the most recent cert was published more than 48
 + * hours ago, then we aren't going to get any consensuses signed with older
 + * keys.) */
 +void
 +trusted_dirs_remove_old_certs(void)
 +{
 +  time_t now = time(NULL);
 +#define DEAD_CERT_LIFETIME (2*24*60*60)
 +#define SUPERSEDED_CERT_LIFETIME (2*24*60*60)
 +  if (!trusted_dir_certs)
 +    return;
 +
 +  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
 +    /* Sort the list from first-published to last-published */
 +    smartlist_sort(cl->certs, compare_certs_by_pubdates);
 +
 +    SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
 +      if (cert_sl_idx == smartlist_len(cl->certs) - 1) {
 +        /* This is the most recently published cert.  Keep it. */
 +        continue;
 +      }
 +      authority_cert_t *next_cert = smartlist_get(cl->certs, cert_sl_idx+1);
 +      const time_t next_cert_published = next_cert->cache_info.published_on;
 +      if (next_cert_published > now) {
 +        /* All later certs are published in the future. Keep everything
 +         * we didn't discard. */
 +        break;
 +      }
 +      int should_remove = 0;
 +      if (cert->expires + DEAD_CERT_LIFETIME < now) {
 +        /* Certificate has been expired for at least DEAD_CERT_LIFETIME.
 +         * Remove it. */
 +        should_remove = 1;
 +      } else if (next_cert_published + SUPERSEDED_CERT_LIFETIME < now) {
 +        /* Certificate has been superseded for OLD_CERT_LIFETIME.
 +         * Remove it.
 +         */
 +        should_remove = 1;
 +      }
 +      if (should_remove) {
 +        SMARTLIST_DEL_CURRENT_KEEPORDER(cl->certs, cert);
 +        authority_cert_free(cert);
 +        trusted_dir_servers_certs_changed = 1;
 +      }
 +    } SMARTLIST_FOREACH_END(cert);
 +
 +  } DIGESTMAP_FOREACH_END;
 +#undef DEAD_CERT_LIFETIME
 +#undef OLD_CERT_LIFETIME
 +
 +  trusted_dirs_flush_certs_to_disk();
 +}
 +
 +/** Return the newest v3 authority certificate whose v3 authority identity key
 + * has digest <b>id_digest</b>.  Return NULL if no such authority is known,
 + * or it has no certificate. */
 +authority_cert_t *
 +authority_cert_get_newest_by_id(const char *id_digest)
 +{
 +  cert_list_t *cl;
 +  authority_cert_t *best = NULL;
 +  if (!trusted_dir_certs ||
 +      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
 +    return NULL;
 +
 +  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
 +  {
 +    if (!best || cert->cache_info.published_on > best->cache_info.published_on)
 +      best = cert;
 +  });
 +  return best;
 +}
 +
 +/** Return the newest v3 authority certificate whose directory signing key has
 + * digest <b>sk_digest</b>. Return NULL if no such certificate is known.
 + */
 +authority_cert_t *
 +authority_cert_get_by_sk_digest(const char *sk_digest)
 +{
 +  authority_cert_t *c;
 +  if (!trusted_dir_certs)
 +    return NULL;
 +
 +  if ((c = get_my_v3_authority_cert()) &&
 +      tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
 +    return c;
 +  if ((c = get_my_v3_legacy_cert()) &&
 +      tor_memeq(c->signing_key_digest, sk_digest, DIGEST_LEN))
 +    return c;
 +
 +  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
 +    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
 +    {
 +      if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
 +        return cert;
 +    });
 +  } DIGESTMAP_FOREACH_END;
 +  return NULL;
 +}
 +
 +/** Return the v3 authority certificate with signing key matching
 + * <b>sk_digest</b>, for the authority with identity digest <b>id_digest</b>.
 + * Return NULL if no such authority is known. */
 +authority_cert_t *
 +authority_cert_get_by_digests(const char *id_digest,
 +                              const char *sk_digest)
 +{
 +  cert_list_t *cl;
 +  if (!trusted_dir_certs ||
 +      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
 +    return NULL;
 +  SMARTLIST_FOREACH(cl->certs, authority_cert_t *, cert,
 +    if (tor_memeq(cert->signing_key_digest, sk_digest, DIGEST_LEN))
 +      return cert; );
 +
 +  return NULL;
 +}
 +
 +/** Add every known authority_cert_t to <b>certs_out</b>. */
 +void
 +authority_cert_get_all(smartlist_t *certs_out)
 +{
 +  tor_assert(certs_out);
 +  if (!trusted_dir_certs)
 +    return;
 +
 +  DIGESTMAP_FOREACH(trusted_dir_certs, key, cert_list_t *, cl) {
 +    SMARTLIST_FOREACH(cl->certs, authority_cert_t *, c,
 +                      smartlist_add(certs_out, c));
 +  } DIGESTMAP_FOREACH_END;
 +}
 +
 +/** Called when an attempt to download a certificate with the authority with
 + * ID <b>id_digest</b> and, if not NULL, signed with key signing_key_digest
 + * fails with HTTP response code <b>status</b>: remember the failure, so we
 + * don't try again immediately. */
 +void
 +authority_cert_dl_failed(const char *id_digest,
 +                         const char *signing_key_digest, int status)
 +{
 +  cert_list_t *cl;
 +  download_status_t *dlstatus = NULL;
 +  char id_digest_str[2*DIGEST_LEN+1];
 +  char sk_digest_str[2*DIGEST_LEN+1];
 +
 +  if (!trusted_dir_certs ||
 +      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
 +    return;
 +
 +  /*
 +   * Are we noting a failed download of the latest cert for the id digest,
 +   * or of a download by (id, signing key) digest pair?
 +   */
 +  if (!signing_key_digest) {
 +    /* Just by id digest */
 +    download_status_failed(&cl->dl_status_by_id, status);
 +  } else {
 +    /* Reset by (id, signing key) digest pair
 +     *
 +     * Look for a download_status_t in the map with this digest
 +     */
 +    dlstatus = dsmap_get(cl->dl_status_map, signing_key_digest);
 +    /* Got one? */
 +    if (dlstatus) {
 +      download_status_failed(dlstatus, status);
 +    } else {
 +      /*
 +       * Do this rather than hex_str(), since hex_str clobbers
 +       * old results and we call twice in the param list.
 +       */
 +      base16_encode(id_digest_str, sizeof(id_digest_str),
 +                    id_digest, DIGEST_LEN);
 +      base16_encode(sk_digest_str, sizeof(sk_digest_str),
 +                    signing_key_digest, DIGEST_LEN);
 +      log_warn(LD_BUG,
 +               "Got failure for cert fetch with (fp,sk) = (%s,%s), with "
 +               "status %d, but knew nothing about the download.",
 +               id_digest_str, sk_digest_str, status);
 +    }
 +  }
 +}
 +
 +static const char *BAD_SIGNING_KEYS[] = {
 +  "09CD84F751FD6E955E0F8ADB497D5401470D697E", // Expires 2015-01-11 16:26:31
 +  "0E7E9C07F0969D0468AD741E172A6109DC289F3C", // Expires 2014-08-12 10:18:26
 +  "57B85409891D3FB32137F642FDEDF8B7F8CDFDCD", // Expires 2015-02-11 17:19:09
 +  "87326329007AF781F587AF5B594E540B2B6C7630", // Expires 2014-07-17 11:10:09
 +  "98CC82342DE8D298CF99D3F1A396475901E0D38E", // Expires 2014-11-10 13:18:56
 +  "9904B52336713A5ADCB13E4FB14DC919E0D45571", // Expires 2014-04-20 20:01:01
 +  "9DCD8E3F1DD1597E2AD476BBA28A1A89F3095227", // Expires 2015-01-16 03:52:30
 +  "A61682F34B9BB9694AC98491FE1ABBFE61923941", // Expires 2014-06-11 09:25:09
 +  "B59F6E99C575113650C99F1C425BA7B20A8C071D", // Expires 2014-07-31 13:22:10
 +  "D27178388FA75B96D37FA36E0B015227DDDBDA51", // Expires 2014-08-04 04:01:57
 +  NULL,
 +};
 +
 +/** Return true iff <b>cert</b> authenticates some atuhority signing key
 + * which, because of the old openssl heartbleed vulnerability, should
 + * never be trusted. */
 +int
 +authority_cert_is_blacklisted(const authority_cert_t *cert)
 +{
 +  char hex_digest[HEX_DIGEST_LEN+1];
 +  int i;
 +  base16_encode(hex_digest, sizeof(hex_digest),
 +                cert->signing_key_digest, sizeof(cert->signing_key_digest));
 +
 +  for (i = 0; BAD_SIGNING_KEYS[i]; ++i) {
 +    if (!strcasecmp(hex_digest, BAD_SIGNING_KEYS[i])) {
 +      return 1;
 +    }
 +  }
 +  return 0;
 +}
 +
 +/** Return true iff when we've been getting enough failures when trying to
 + * download the certificate with ID digest <b>id_digest</b> that we're willing
 + * to start bugging the user about it. */
 +int
 +authority_cert_dl_looks_uncertain(const char *id_digest)
 +{
 +#define N_AUTH_CERT_DL_FAILURES_TO_BUG_USER 2
 +  cert_list_t *cl;
 +  int n_failures;
 +  if (!trusted_dir_certs ||
 +      !(cl = digestmap_get(trusted_dir_certs, id_digest)))
 +    return 0;
 +
 +  n_failures = download_status_get_n_failures(&cl->dl_status_by_id);
 +  return n_failures >= N_AUTH_CERT_DL_FAILURES_TO_BUG_USER;
 +}
 +
 +/* Fetch the authority certificates specified in resource.
 + * If we are a bridge client, and node is a configured bridge, fetch from node
 + * using dir_hint as the fingerprint. Otherwise, if rs is not NULL, fetch from
 + * rs. Otherwise, fetch from a random directory mirror. */
 +static void
 +authority_certs_fetch_resource_impl(const char *resource,
 +                                    const char *dir_hint,
 +                                    const node_t *node,
 +                                    const routerstatus_t *rs)
 +{
 +  const or_options_t *options = get_options();
 +  int get_via_tor = purpose_needs_anonymity(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
 +                                            resource);
 +
 +  /* Make sure bridge clients never connect to anything but a bridge */
 +  if (options->UseBridges) {
 +    if (node && !node_is_a_configured_bridge(node)) {
 +      /* If we're using bridges, and node is not a bridge, use a 3-hop path. */
 +      get_via_tor = 1;
 +    } else if (!node) {
 +      /* If we're using bridges, and there's no node, use a 3-hop path. */
 +      get_via_tor = 1;
 +    }
 +  }
 +
 +  const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS
 +                                                    : DIRIND_ONEHOP;
 +
 +  directory_request_t *req = NULL;
 +  /* If we've just downloaded a consensus from a bridge, re-use that
 +   * bridge */
 +  if (options->UseBridges && node && node->ri && !get_via_tor) {
 +    /* clients always make OR connections to bridges */
 +    tor_addr_port_t or_ap;
 +    /* we are willing to use a non-preferred address if we need to */
 +    fascist_firewall_choose_address_node(node, FIREWALL_OR_CONNECTION, 0,
 +                                         &or_ap);
 +
 +    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
 +    directory_request_set_or_addr_port(req, &or_ap);
 +    if (dir_hint)
 +      directory_request_set_directory_id_digest(req, dir_hint);
 +  } else if (rs) {
 +    /* And if we've just downloaded a consensus from a directory, re-use that
 +     * directory */
 +    req = directory_request_new(DIR_PURPOSE_FETCH_CERTIFICATE);
 +    directory_request_set_routerstatus(req, rs);
 +  }
 +
 +  if (req) {
 +    /* We've set up a request object -- fill in the other request fields, and
 +     * send the request.  */
 +    directory_request_set_indirection(req, indirection);
 +    directory_request_set_resource(req, resource);
 +    directory_initiate_request(req);
 +    directory_request_free(req);
 +    return;
 +  }
 +
 +  /* Otherwise, we want certs from a random fallback or directory
 +   * mirror, because they will almost always succeed. */
 +  directory_get_from_dirserver(DIR_PURPOSE_FETCH_CERTIFICATE, 0,
 +                               resource, PDS_RETRY_IF_NO_SERVERS,
 +                               DL_WANT_ANY_DIRSERVER);
 +}
 +
 +/** Try to download any v3 authority certificates that we may be missing.  If
 + * <b>status</b> is provided, try to get all the ones that were used to sign
 + * <b>status</b>.  Additionally, try to have a non-expired certificate for
 + * every V3 authority in trusted_dir_servers.  Don't fetch certificates we
 + * already have.
 + *
 + * If dir_hint is non-NULL, it's the identity digest for a directory that
 + * we've just successfully retrieved a consensus or certificates from, so try
 + * it first to fetch any missing certificates.
 + **/
 +void
 +authority_certs_fetch_missing(networkstatus_t *status, time_t now,
 +                              const char *dir_hint)
 +{
 +  /*
 +   * The pending_id digestmap tracks pending certificate downloads by
 +   * identity digest; the pending_cert digestmap tracks pending downloads
 +   * by (identity digest, signing key digest) pairs.
 +   */
 +  digestmap_t *pending_id;
 +  fp_pair_map_t *pending_cert;
 +  /*
 +   * The missing_id_digests smartlist will hold a list of id digests
 +   * we want to fetch the newest cert for; the missing_cert_digests
 +   * smartlist will hold a list of fp_pair_t with an identity and
 +   * signing key digest.
 +   */
 +  smartlist_t *missing_cert_digests, *missing_id_digests;
 +  char *resource = NULL;
 +  cert_list_t *cl;
 +  const or_options_t *options = get_options();
 +  const int keep_unknown = we_want_to_fetch_unknown_auth_certs(options);
 +  fp_pair_t *fp_tmp = NULL;
 +  char id_digest_str[2*DIGEST_LEN+1];
 +  char sk_digest_str[2*DIGEST_LEN+1];
 +
 +  if (should_delay_dir_fetches(options, NULL))
 +    return;
 +
 +  pending_cert = fp_pair_map_new();
 +  pending_id = digestmap_new();
 +  missing_cert_digests = smartlist_new();
 +  missing_id_digests = smartlist_new();
 +
 +  /*
 +   * First, we get the lists of already pending downloads so we don't
 +   * duplicate effort.
 +   */
 +  list_pending_downloads(pending_id, NULL,
 +                         DIR_PURPOSE_FETCH_CERTIFICATE, "fp/");
 +  list_pending_fpsk_downloads(pending_cert);
 +
 +  /*
 +   * Now, we download any trusted authority certs we don't have by
 +   * identity digest only.  This gets the latest cert for that authority.
 +   */
 +  SMARTLIST_FOREACH_BEGIN(router_get_trusted_dir_servers(),
 +                          dir_server_t *, ds) {
 +    int found = 0;
 +    if (!(ds->type & V3_DIRINFO))
 +      continue;
 +    if (smartlist_contains_digest(missing_id_digests,
 +                                  ds->v3_identity_digest))
 +      continue;
 +    cl = get_cert_list(ds->v3_identity_digest);
 +    SMARTLIST_FOREACH_BEGIN(cl->certs, authority_cert_t *, cert) {
 +      if (now < cert->expires) {
 +        /* It's not expired, and we weren't looking for something to
 +         * verify a consensus with.  Call it done. */
 +        download_status_reset(&(cl->dl_status_by_id));
 +        /* No sense trying to download it specifically by signing key hash */
 +        download_status_reset_by_sk_in_cl(cl, cert->signing_key_digest);
 +        found = 1;
 +        break;
 +      }
 +    } SMARTLIST_FOREACH_END(cert);
 +    if (!found &&
 +        download_status_is_ready(&(cl->dl_status_by_id), now) &&
 +        !digestmap_get(pending_id, ds->v3_identity_digest)) {
 +      log_info(LD_DIR,
 +               "No current certificate known for authority %s "
 +               "(ID digest %s); launching request.",
 +               ds->nickname, hex_str(ds->v3_identity_digest, DIGEST_LEN));
 +      smartlist_add(missing_id_digests, ds->v3_identity_digest);
 +    }
 +  } SMARTLIST_FOREACH_END(ds);
 +
 +  /*
 +   * Next, if we have a consensus, scan through it and look for anything
 +   * signed with a key from a cert we don't have.  Those get downloaded
 +   * by (fp,sk) pair, but if we don't know any certs at all for the fp
 +   * (identity digest), and it's one of the trusted dir server certs
 +   * we started off above or a pending download in pending_id, don't
 +   * try to get it yet.  Most likely, the one we'll get for that will
 +   * have the right signing key too, and we'd just be downloading
 +   * redundantly.
 +   */
 +  if (status) {
 +    SMARTLIST_FOREACH_BEGIN(status->voters, networkstatus_voter_info_t *,
 +                            voter) {
 +      if (!smartlist_len(voter->sigs))
 +        continue; /* This authority never signed this consensus, so don't
 +                   * go looking for a cert with key digest 0000000000. */
 +      if (!keep_unknown &&
 +          !trusteddirserver_get_by_v3_auth_digest(voter->identity_digest))
 +        continue; /* We don't want unknown certs, and we don't know this
 +                   * authority.*/
 +
 +      /*
 +       * If we don't know *any* cert for this authority, and a download by ID
 +       * is pending or we added it to missing_id_digests above, skip this
 +       * one for now to avoid duplicate downloads.
 +       */
 +      cl = get_cert_list(voter->identity_digest);
 +      if (smartlist_len(cl->certs) == 0) {
 +        /* We have no certs at all for this one */
 +
 +        /* Do we have a download of one pending? */
 +        if (digestmap_get(pending_id, voter->identity_digest))
 +          continue;
 +
 +        /*
 +         * Are we about to launch a download of one due to the trusted
 +         * dir server check above?
 +         */
 +        if (smartlist_contains_digest(missing_id_digests,
 +                                      voter->identity_digest))
 +          continue;
 +      }
 +
 +      SMARTLIST_FOREACH_BEGIN(voter->sigs, document_signature_t *, sig) {
 +        authority_cert_t *cert =
 +          authority_cert_get_by_digests(voter->identity_digest,
 +                                        sig->signing_key_digest);
 +        if (cert) {
 +          if (now < cert->expires)
 +            download_status_reset_by_sk_in_cl(cl, sig->signing_key_digest);
 +          continue;
 +        }
 +        if (download_status_is_ready_by_sk_in_cl(
 +              cl, sig->signing_key_digest, now) &&
 +            !fp_pair_map_get_by_digests(pending_cert,
 +                                        voter->identity_digest,
 +                                        sig->signing_key_digest)) {
 +          /*
 +           * Do this rather than hex_str(), since hex_str clobbers
 +           * old results and we call twice in the param list.
 +           */
 +          base16_encode(id_digest_str, sizeof(id_digest_str),
 +                        voter->identity_digest, DIGEST_LEN);
 +          base16_encode(sk_digest_str, sizeof(sk_digest_str),
 +                        sig->signing_key_digest, DIGEST_LEN);
 +
 +          if (voter->nickname) {
 +            log_info(LD_DIR,
 +                     "We're missing a certificate from authority %s "
 +                     "(ID digest %s) with signing key %s: "
 +                     "launching request.",
 +                     voter->nickname, id_digest_str, sk_digest_str);
 +          } else {
 +            log_info(LD_DIR,
 +                     "We're missing a certificate from authority ID digest "
 +                     "%s with signing key %s: launching request.",
 +                     id_digest_str, sk_digest_str);
 +          }
 +
 +          /* Allocate a new fp_pair_t to append */
 +          fp_tmp = tor_malloc(sizeof(*fp_tmp));
 +          memcpy(fp_tmp->first, voter->identity_digest, sizeof(fp_tmp->first));
 +          memcpy(fp_tmp->second, sig->signing_key_digest,
 +                 sizeof(fp_tmp->second));
 +          smartlist_add(missing_cert_digests, fp_tmp);
 +        }
 +      } SMARTLIST_FOREACH_END(sig);
 +    } SMARTLIST_FOREACH_END(voter);
 +  }
 +
 +  /* Bridge clients look up the node for the dir_hint */
 +  const node_t *node = NULL;
 +  /* All clients, including bridge clients, look up the routerstatus for the
 +   * dir_hint */
 +  const routerstatus_t *rs = NULL;
 +
 +  /* If we still need certificates, try the directory that just successfully
 +   * served us a consensus or certificates.
 +   * As soon as the directory fails to provide additional certificates, we try
 +   * another, randomly selected directory. This avoids continual retries.
 +   * (We only ever have one outstanding request per certificate.)
 +   */
 +  if (dir_hint) {
 +    if (options->UseBridges) {
 +      /* Bridge clients try the nodelist. If the dir_hint is from an authority,
 +       * or something else fetched over tor, we won't find the node here, but
 +       * we will find the rs. */
 +      node = node_get_by_id(dir_hint);
 +    }
 +
 +    /* All clients try the consensus routerstatus, then the fallback
 +     * routerstatus */
 +    rs = router_get_consensus_status_by_id(dir_hint);
 +    if (!rs) {
 +      /* This will also find authorities */
 +      const dir_server_t *ds = router_get_fallback_dirserver_by_digest(
 +                                                                    dir_hint);
 +      if (ds) {
 +        rs = &ds->fake_status;
 +      }
 +    }
 +
 +    if (!node && !rs) {
 +      log_warn(LD_BUG, "Directory %s delivered a consensus, but %s"
 +               "no routerstatus could be found for it.",
 +               options->UseBridges ? "no node and " : "",
 +               hex_str(dir_hint, DIGEST_LEN));
 +    }
 +  }
 +
 +  /* Do downloads by identity digest */
 +  if (smartlist_len(missing_id_digests) > 0) {
 +    int need_plus = 0;
 +    smartlist_t *fps = smartlist_new();
 +
 +    smartlist_add_strdup(fps, "fp/");
 +
 +    SMARTLIST_FOREACH_BEGIN(missing_id_digests, const char *, d) {
 +      char *fp = NULL;
 +
 +      if (digestmap_get(pending_id, d))
 +        continue;
 +
 +      base16_encode(id_digest_str, sizeof(id_digest_str),
 +                    d, DIGEST_LEN);
 +
 +      if (need_plus) {
 +        tor_asprintf(&fp, "+%s", id_digest_str);
 +      } else {
 +        /* No need for tor_asprintf() in this case; first one gets no '+' */
 +        fp = tor_strdup(id_digest_str);
 +        need_plus = 1;
 +      }
 +
 +      smartlist_add(fps, fp);
 +    } SMARTLIST_FOREACH_END(d);
 +
 +    if (smartlist_len(fps) > 1) {
 +      resource = smartlist_join_strings(fps, "", 0, NULL);
 +      /* node and rs are directories that just gave us a consensus or
 +       * certificates */
 +      authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
 +      tor_free(resource);
 +    }
 +    /* else we didn't add any: they were all pending */
 +
 +    SMARTLIST_FOREACH(fps, char *, cp, tor_free(cp));
 +    smartlist_free(fps);
 +  }
 +
 +  /* Do downloads by identity digest/signing key pair */
 +  if (smartlist_len(missing_cert_digests) > 0) {
 +    int need_plus = 0;
 +    smartlist_t *fp_pairs = smartlist_new();
 +
 +    smartlist_add_strdup(fp_pairs, "fp-sk/");
 +
 +    SMARTLIST_FOREACH_BEGIN(missing_cert_digests, const fp_pair_t *, d) {
 +      char *fp_pair = NULL;
 +
 +      if (fp_pair_map_get(pending_cert, d))
 +        continue;
 +
 +      /* Construct string encodings of the digests */
 +      base16_encode(id_digest_str, sizeof(id_digest_str),
 +                    d->first, DIGEST_LEN);
 +      base16_encode(sk_digest_str, sizeof(sk_digest_str),
 +                    d->second, DIGEST_LEN);
 +
 +      /* Now tor_asprintf() */
 +      if (need_plus) {
 +        tor_asprintf(&fp_pair, "+%s-%s", id_digest_str, sk_digest_str);
 +      } else {
 +        /* First one in the list doesn't get a '+' */
 +        tor_asprintf(&fp_pair, "%s-%s", id_digest_str, sk_digest_str);
 +        need_plus = 1;
 +      }
 +
 +      /* Add it to the list of pairs to request */
 +      smartlist_add(fp_pairs, fp_pair);
 +    } SMARTLIST_FOREACH_END(d);
 +
 +    if (smartlist_len(fp_pairs) > 1) {
 +      resource = smartlist_join_strings(fp_pairs, "", 0, NULL);
 +      /* node and rs are directories that just gave us a consensus or
 +       * certificates */
 +      authority_certs_fetch_resource_impl(resource, dir_hint, node, rs);
 +      tor_free(resource);
 +    }
 +    /* else they were all pending */
 +
 +    SMARTLIST_FOREACH(fp_pairs, char *, p, tor_free(p));
 +    smartlist_free(fp_pairs);
 +  }
 +
 +  smartlist_free(missing_id_digests);
 +  SMARTLIST_FOREACH(missing_cert_digests, fp_pair_t *, p, tor_free(p));
 +  smartlist_free(missing_cert_digests);
 +  digestmap_free(pending_id, NULL);
 +  fp_pair_map_free(pending_cert, NULL);
 +}
 +
 +void
 +authcert_free_all(void)
 +{
 +  if (trusted_dir_certs) {
 +    digestmap_free(trusted_dir_certs, cert_list_free_void);
 +    trusted_dir_certs = NULL;
 +  }
 +}
 +
 +/** Free storage held in <b>cert</b>. */
 +void
 +authority_cert_free_(authority_cert_t *cert)
 +{
 +  if (!cert)
 +    return;
 +
 +  tor_free(cert->cache_info.signed_descriptor_body);
 +  crypto_pk_free(cert->signing_key);
 +  crypto_pk_free(cert->identity_key);
 +
 +  tor_free(cert);
 +}
 +
 +/** For every certificate we are currently downloading by (identity digest,
 + * signing key digest) pair, set result[fp_pair] to (void *1).
 + */
 +static void
 +list_pending_fpsk_downloads(fp_pair_map_t *result)
 +{
 +  const char *pfx = "fp-sk/";
 +  smartlist_t *tmp;
 +  smartlist_t *conns;
 +  const char *resource;
 +
 +  tor_assert(result);
 +
 +  tmp = smartlist_new();
 +  conns = get_connection_array();
 +
 +  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
 +    if (conn->type == CONN_TYPE_DIR &&
 +        conn->purpose == DIR_PURPOSE_FETCH_CERTIFICATE &&
 +        !conn->marked_for_close) {
 +      resource = TO_DIR_CONN(conn)->requested_resource;
 +      if (!strcmpstart(resource, pfx))
 +        dir_split_resource_into_fingerprint_pairs(resource + strlen(pfx),
 +                                                  tmp);
 +    }
 +  } SMARTLIST_FOREACH_END(conn);
 +
 +  SMARTLIST_FOREACH_BEGIN(tmp, fp_pair_t *, fp) {
 +    fp_pair_map_set(result, fp, (void*)1);
 +    tor_free(fp);
 +  } SMARTLIST_FOREACH_END(fp);
 +
 +  smartlist_free(tmp);
 +}
diff --cc src/feature/nodelist/networkstatus.c
index de2451b79,97a8779be..ec1a69b9e
--- a/src/feature/nodelist/networkstatus.c
+++ b/src/feature/nodelist/networkstatus.c
@@@ -2109,14 -2106,14 +2117,15 @@@ networkstatus_set_current_consensus(con
    }
  
    if (we_want_to_fetch_flavor(options, flav)) {
 -    dirserv_set_cached_consensus_networkstatus(consensus,
 -                                               consensus_len,
 -                                               flavor,
 -                                               &c->digests,
 -                                               c->digest_sha3_as_signed,
 -                                               c->valid_after);
      if (dir_server_mode(get_options())) {
 +      dirserv_set_cached_consensus_networkstatus(consensus,
++                                                 consensus_len,
 +                                                 flavor,
 +                                                 &c->digests,
 +                                                 c->digest_sha3_as_signed,
 +                                                 c->valid_after);
 +
-       consdiffmgr_add_consensus(consensus, c);
+       consdiffmgr_add_consensus(consensus, consensus_len, c);
      }
    }
  



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits