[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] [tor/maint-0.2.2] Merge branch 'maint-0.2.1' into maint-0.2.2



commit 9b745cdbf9cd7384e44e18bf40a3d2c9becbc345
Merge: d37660d 28de4d8
Author: Roger Dingledine <arma@xxxxxxxxxxxxxx>
Date:   Fri Feb 11 01:20:47 2011 -0500

    Merge branch 'maint-0.2.1' into maint-0.2.2

 changes/bug1074-part2 |    6 ++++++
 src/or/directory.c    |    7 ++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --combined src/or/directory.c
index 7360a47,7150fce..f8d587f
--- a/src/or/directory.c
+++ b/src/or/directory.c
@@@ -4,26 -4,6 +4,26 @@@
  /* See LICENSE for licensing information */
  
  #include "or.h"
 +#include "buffers.h"
 +#include "circuitbuild.h"
 +#include "config.h"
 +#include "connection.h"
 +#include "connection_edge.h"
 +#include "control.h"
 +#include "directory.h"
 +#include "dirserv.h"
 +#include "dirvote.h"
 +#include "geoip.h"
 +#include "main.h"
 +#include "networkstatus.h"
 +#include "policies.h"
 +#include "rendclient.h"
 +#include "rendcommon.h"
 +#include "rephist.h"
 +#include "router.h"
 +#include "routerlist.h"
 +#include "routerparse.h"
 +
  #if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
  #ifndef OPENBSD
  #include <malloc.h>
@@@ -67,10 -47,8 +67,10 @@@ static void http_set_address_origin(con
  static void connection_dir_download_networkstatus_failed(
                                 dir_connection_t *conn, int status_code);
  static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
 +static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn);
  static void connection_dir_download_cert_failed(
                                 dir_connection_t *conn, int status_code);
 +static void connection_dir_retry_bridges(smartlist_t *descs);
  static void dir_networkstatus_download_failed(smartlist_t *failed,
                                                int status_code);
  static void dir_routerdesc_download_failed(smartlist_t *failed,
@@@ -114,19 -92,17 +114,19 @@@ static void directory_initiate_command_
  #define ROUTERDESC_CACHE_LIFETIME (30*60)
  #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
  #define ROBOTS_CACHE_LIFETIME (24*60*60)
 +#define MICRODESC_CACHE_LIFETIME (48*60*60)
  
  /********* END VARIABLES ************/
  
 -/** Return true iff the directory purpose 'purpose' must use an
 - * anonymous connection to a directory. */
 +/** Return true iff the directory purpose <b>dir_purpose</b> (and if it's
 + * fetching descriptors, it's fetching them for <b>router_purpose</b>)
 + * must use an anonymous connection to a directory. */
  static int
  purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose)
  {
    if (get_options()->AllDirActionsPrivate)
      return 1;
 -  if (router_purpose == ROUTER_PURPOSE_BRIDGE && has_completed_circuit)
 +  if (router_purpose == ROUTER_PURPOSE_BRIDGE && can_complete_circuit)
      return 1; /* if no circuits yet, we may need this info to bootstrap. */
    if (dir_purpose == DIR_PURPOSE_UPLOAD_DIR ||
        dir_purpose == DIR_PURPOSE_UPLOAD_VOTE ||
@@@ -254,7 -230,7 +254,7 @@@ directories_have_accepted_server_descri
  
  /** Start a connection to every suitable directory authority, using
   * connection purpose 'purpose' and uploading the payload 'payload'
 - * (length 'payload_len').  The purpose should be one of
 + * (length 'payload_len').  dir_purpose should be one of
   * 'DIR_PURPOSE_UPLOAD_DIR' or 'DIR_PURPOSE_UPLOAD_RENDDESC'.
   *
   * <b>type</b> specifies what sort of dir authorities (V1, V2,
@@@ -584,7 -560,7 +584,7 @@@ connection_dir_request_failed(dir_conne
    if (directory_conn_is_self_reachability_test(conn)) {
      return; /* this was a test fetch. don't retry. */
    }
 -  if (entry_list_can_grow(get_options()))
 +  if (!entry_list_is_constrained(get_options()))
      router_set_status(conn->identity_digest, 0); /* don't try him again */
    if (conn->_base.purpose == DIR_PURPOSE_FETCH_V2_NETWORKSTATUS) {
      log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
@@@ -594,8 -570,6 +594,8 @@@
               conn->_base.purpose == DIR_PURPOSE_FETCH_EXTRAINFO) {
      log_info(LD_DIR, "Giving up on directory server at '%s'; retrying",
               conn->_base.address);
 +    if (conn->router_purpose == ROUTER_PURPOSE_BRIDGE)
 +      connection_dir_bridge_routerdesc_failed(conn);
      connection_dir_download_routerdesc_failed(conn);
    } else if (conn->_base.purpose == DIR_PURPOSE_FETCH_CONSENSUS) {
      networkstatus_consensus_download_failed(0);
@@@ -640,7 -614,7 +640,7 @@@ connection_dir_download_networkstatus_f
       * failed, and possibly retry them later.*/
      smartlist_t *failed = smartlist_create();
      dir_split_resource_into_fingerprints(conn->requested_resource+3,
 -                                         failed, NULL, 0, 0);
 +                                         failed, NULL, 0);
      if (smartlist_len(failed)) {
        dir_networkstatus_download_failed(failed, status_code);
        SMARTLIST_FOREACH(failed, char *, cp, tor_free(cp));
@@@ -649,24 -623,6 +649,24 @@@
    }
  }
  
 +/** Helper: Attempt to fetch directly the descriptors of each bridge
 + * listed in <b>failed</b>.
 + */
 +static void
 +connection_dir_retry_bridges(smartlist_t *descs)
 +{
 +  char digest[DIGEST_LEN];
 +  SMARTLIST_FOREACH(descs, const char *, cp,
 +  {
 +    if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) {
 +      log_warn(LD_BUG, "Malformed fingerprint in list: %s",
 +              escaped(cp));
 +      continue;
 +    }
 +    retry_bridge_descriptor_fetch_directly(digest);
 +  });
 +}
 +
  /** Called when an attempt to download one or more router descriptors
   * or extra-info documents on connection <b>conn</b> failed.
   */
@@@ -684,33 -640,6 +684,33 @@@ connection_dir_download_routerdesc_fail
    (void) conn;
  }
  
 +/** Called when an attempt to download a bridge's routerdesc from
 + * one of the authorities failed due to a network error. If
 + * possible attempt to download descriptors from the bridge directly.
 + */
 +static void
 +connection_dir_bridge_routerdesc_failed(dir_connection_t *conn)
 +{
 +  smartlist_t *which = NULL;
 +
 +  /* Requests for bridge descriptors are in the form 'fp/', so ignore
 +     anything else. */
 +  if (!conn->requested_resource || strcmpstart(conn->requested_resource,"fp/"))
 +    return;
 +
 +  which = smartlist_create();
 +  dir_split_resource_into_fingerprints(conn->requested_resource
 +                                        + strlen("fp/"),
 +                                       which, NULL, 0);
 +
 +  tor_assert(conn->_base.purpose != DIR_PURPOSE_FETCH_EXTRAINFO);
 +  if (smartlist_len(which)) {
 +    connection_dir_retry_bridges(which);
 +    SMARTLIST_FOREACH(which, char *, cp, tor_free(cp));
 +  }
 +  smartlist_free(which);
 +}
 +
  /** Called when an attempt to fetch a certificate fails. */
  static void
  connection_dir_download_cert_failed(dir_connection_t *conn, int status)
@@@ -722,7 -651,7 +722,7 @@@
      return;
    failed = smartlist_create();
    dir_split_resource_into_fingerprints(conn->requested_resource+3,
 -                                       failed, NULL, 1, 0);
 +                                       failed, NULL, DSR_HEX);
    SMARTLIST_FOREACH(failed, char *, cp,
    {
      authority_cert_dl_failed(cp, status);
@@@ -738,7 -667,7 +738,7 @@@
   * 1) If or_port is 0, or it's a direct conn and or_port is firewalled
   *    or we're a dir mirror, no.
   * 2) If we prefer to avoid begindir conns, and we're not fetching or
 - * publishing a bridge relay descriptor, no.
 + *    publishing a bridge relay descriptor, no.
   * 3) Else yes.
   */
  static int
@@@ -817,15 -746,6 +817,15 @@@ directory_initiate_command_rend(const c
  
    log_debug(LD_DIR, "Initiating %s", dir_conn_purpose_to_string(dir_purpose));
  
 +  /* ensure that we don't make direct connections when a SOCKS server is
 +   * configured. */
 +  if (!anonymized_connection && !use_begindir && !options->HTTPProxy &&
 +      (options->Socks4Proxy || options->Socks5Proxy)) {
 +    log_warn(LD_DIR, "Cannot connect to a directory server through a "
 +                     "SOCKS proxy!");
 +    return;
 +  }
 +
    conn = dir_connection_new(AF_INET);
  
    /* set up conn so it's got all the data we need to remember */
@@@ -850,9 -770,9 +850,9 @@@
    if (!anonymized_connection && !use_begindir) {
      /* then we want to connect to dirport directly */
  
 -    if (options->HttpProxy) {
 -      tor_addr_from_ipv4h(&addr, options->HttpProxyAddr);
 -      dir_port = options->HttpProxyPort;
 +    if (options->HTTPProxy) {
 +      tor_addr_copy(&addr, &options->HTTPProxyAddr);
 +      dir_port = options->HTTPProxyPort;
      }
  
      switch (connection_connect(TO_CONN(conn), conn->_base.address, &addr,
@@@ -873,7 -793,7 +873,7 @@@
                                 payload, payload_len,
                                 supports_conditional_consensus,
                                 if_modified_since);
 -        connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE);
 +        connection_watch_events(TO_CONN(conn), READ_EVENT | WRITE_EVENT);
          /* writable indicates finish, readable indicates broken link,
             error indicates broken link in windowsland. */
      }
@@@ -912,7 -832,7 +912,7 @@@
                             payload, payload_len,
                             supports_conditional_consensus,
                             if_modified_since);
 -    connection_watch_events(TO_CONN(conn), EV_READ | EV_WRITE);
 +    connection_watch_events(TO_CONN(conn), READ_EVENT|WRITE_EVENT);
      connection_start_reading(TO_CONN(linked_conn));
    }
  }
@@@ -960,7 -880,7 +960,7 @@@ directory_get_consensus_url(int support
  
    if (supports_conditional_consensus) {
      char *authority_id_list;
 -    smartlist_t *authority_digets = smartlist_create();
 +    smartlist_t *authority_digests = smartlist_create();
  
      SMARTLIST_FOREACH(router_get_trusted_dir_servers(),
                        trusted_dir_server_t *, ds,
@@@ -972,10 -892,10 +972,10 @@@
          hex = tor_malloc(2*CONDITIONAL_CONSENSUS_FPR_LEN+1);
          base16_encode(hex, 2*CONDITIONAL_CONSENSUS_FPR_LEN+1,
                        ds->v3_identity_digest, CONDITIONAL_CONSENSUS_FPR_LEN);
 -        smartlist_add(authority_digets, hex);
 +        smartlist_add(authority_digests, hex);
        });
 -    smartlist_sort(authority_digets, _compare_strs);
 -    authority_id_list = smartlist_join_strings(authority_digets,
 +    smartlist_sort(authority_digests, _compare_strs);
 +    authority_id_list = smartlist_join_strings(authority_digests,
                                                 "+", 0, NULL);
  
      len = strlen(authority_id_list)+64;
@@@ -983,8 -903,8 +983,8 @@@
      tor_snprintf(url, len, "/tor/status-vote/current/consensus/%s.z",
                   authority_id_list);
  
 -    SMARTLIST_FOREACH(authority_digets, char *, cp, tor_free(cp));
 -    smartlist_free(authority_digets);
 +    SMARTLIST_FOREACH(authority_digests, char *, cp, tor_free(cp));
 +    smartlist_free(authority_digests);
      tor_free(authority_id_list);
    } else {
      url = tor_strdup("/tor/status-vote/current/consensus.z");
@@@ -993,7 -913,7 +993,7 @@@
  }
  
  /** Queue an appropriate HTTP command on conn-\>outbuf.  The other args
 - * are as in directory_initiate_command.
 + * are as in directory_initiate_command().
   */
  static void
  directory_send_command(dir_connection_t *conn,
@@@ -1036,9 -956,9 +1036,9 @@@
    }
  
    /* come up with some proxy lines, if we're using one. */
 -  if (direct && get_options()->HttpProxy) {
 +  if (direct && get_options()->HTTPProxy) {
      char *base64_authenticator=NULL;
 -    const char *authenticator = get_options()->HttpProxyAuthenticator;
 +    const char *authenticator = get_options()->HTTPProxyAuthenticator;
  
      tor_snprintf(proxystring, sizeof(proxystring),"http://%s";, hoststring);
      if (authenticator) {
@@@ -1129,10 -1049,31 +1129,10 @@@
        httpcommand = "POST";
        url = tor_strdup("/tor/post/consensus-signature");
        break;
 -    case DIR_PURPOSE_FETCH_RENDDESC:
 -      tor_assert(resource);
 -      tor_assert(!payload);
 -
 -      /* this must be true or we wouldn't be doing the lookup */
 -      tor_assert(strlen(resource) <= REND_SERVICE_ID_LEN_BASE32);
 -      /* This breaks the function abstraction. */
 -      conn->rend_data = tor_malloc_zero(sizeof(rend_data_t));
 -      strlcpy(conn->rend_data->onion_address, resource,
 -              sizeof(conn->rend_data->onion_address));
 -      conn->rend_data->rend_desc_version = 0;
 -
 -      httpcommand = "GET";
 -      /* Request the most recent versioned descriptor. */
 -      // (XXXX We were going to switch this to fetch rendezvous1 descriptors,
 -      // but that never got testing, and it wasn't a good design.)
 -      len = strlen(resource)+32;
 -      url = tor_malloc(len);
 -      tor_snprintf(url, len, "/tor/rendezvous/%s", resource);
 -      break;
      case DIR_PURPOSE_FETCH_RENDDESC_V2:
        tor_assert(resource);
        tor_assert(strlen(resource) <= REND_DESC_ID_V2_LEN_BASE32);
        tor_assert(!payload);
 -      conn->rend_data->rend_desc_version = 2;
        httpcommand = "GET";
        len = strlen(resource) + 32;
        url = tor_malloc(len);
@@@ -1221,7 -1162,7 +1221,7 @@@ parse_http_url(const char *headers, cha
      if (s-tmp >= 3 && !strcmpstart(tmp,"://")) {
        tmp = strchr(tmp+3, '/');
        if (tmp && tmp < s) {
 -        log_debug(LD_DIR,"Skipping over 'http[s]://hostname' string");
 +        log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string");
          start = tmp;
        }
      }
@@@ -1527,9 -1468,10 +1527,10 @@@ connection_dir_client_reached_eof(dir_c
               delta>0 ? "ahead" : "behind", dbuf,
               delta>0 ? "behind" : "ahead");
        skewed = 1; /* don't check the recommended-versions line */
-       control_event_general_status(trusted ? LOG_WARN : LOG_NOTICE,
-                                "CLOCK_SKEW SKEW=%ld SOURCE=DIRSERV:%s:%d",
-                                delta, conn->_base.address, conn->_base.port);
+       if (trusted)
+         control_event_general_status(LOG_WARN,
+                                  "CLOCK_SKEW SKEW=%ld SOURCE=DIRSERV:%s:%d",
+                                  delta, conn->_base.address, conn->_base.port);
      } else {
        log_debug(LD_HTTP, "Time on received directory is within tolerance; "
                  "we are %ld seconds skewed.  (That's okay.)", delta);
@@@ -1537,22 -1479,21 +1538,22 @@@
    }
    (void) skewed; /* skewed isn't used yet. */
  
 -  if (status_code == 503 && body_len < 16) {
 -    routerstatus_t *rs;
 -    trusted_dir_server_t *ds;
 -    log_info(LD_DIR,"Received http status code %d (%s) from server "
 -             "'%s:%d'. I'll try again soon.",
 -             status_code, escaped(reason), conn->_base.address,
 -             conn->_base.port);
 -    if ((rs = router_get_consensus_status_by_id(conn->identity_digest)))
 -      rs->last_dir_503_at = now;
 -    if ((ds = router_get_trusteddirserver_by_digest(conn->identity_digest)))
 -      ds->fake_status.last_dir_503_at = now;
 +  if (status_code == 503) {
 +    if (body_len < 16) {
 +      routerstatus_t *rs;
 +      trusted_dir_server_t *ds;
 +      log_info(LD_DIR,"Received http status code %d (%s) from server "
 +               "'%s:%d'. I'll try again soon.",
 +               status_code, escaped(reason), conn->_base.address,
 +               conn->_base.port);
 +      if ((rs = router_get_consensus_status_by_id(conn->identity_digest)))
 +        rs->last_dir_503_at = now;
 +      if ((ds = router_get_trusteddirserver_by_digest(conn->identity_digest)))
 +        ds->fake_status.last_dir_503_at = now;
  
 -    tor_free(body); tor_free(headers); tor_free(reason);
 -    return -1;
 -  } else if (status_code == 503) {
 +      tor_free(body); tor_free(headers); tor_free(reason);
 +      return -1;
 +    }
      /* XXXX022 Remove this once every server with bug 539 is obsolete. */
      log_info(LD_DIR, "Server at '%s:%d' sent us a 503 response, but included "
               "a body anyway.  We'll pretend it gave us a 200.",
@@@ -1624,7 -1565,7 +1625,7 @@@
      v2_networkstatus_source_t source;
      char *cp;
      log_info(LD_DIR,"Received networkstatus objects (size %d) from server "
 -             "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port);
 +             "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port);
      if (status_code != 200) {
        log_warn(LD_DIR,
             "Received http status code %d (%s) from server "
@@@ -1640,7 -1581,7 +1641,7 @@@
        source = NS_FROM_DIR_BY_FP;
        which = smartlist_create();
        dir_split_resource_into_fingerprints(conn->requested_resource+3,
 -                                           which, NULL, 0, 0);
 +                                           which, NULL, 0);
      } else if (conn->requested_resource &&
                 !strcmpstart(conn->requested_resource, "all")) {
        source = NS_FROM_DIR_ALL;
@@@ -1698,8 -1639,8 +1699,8 @@@
        return -1;
      }
      log_info(LD_DIR,"Received consensus directory (size %d) from server "
 -             "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port);
 -    if ((r=networkstatus_set_current_consensus(body, 0))<0) {
 +             "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port);
 +    if ((r=networkstatus_set_current_consensus(body, "ns", 0))<0) {
        log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
               "Unable to load consensus directory downloaded from "
               "server '%s:%d'. I'll try again soon.",
@@@ -1726,11 -1667,9 +1727,11 @@@
        return -1;
      }
      log_info(LD_DIR,"Received authority certificates (size %d) from server "
 -             "'%s:%d'",(int) body_len, conn->_base.address, conn->_base.port);
 +             "'%s:%d'", (int)body_len, conn->_base.address, conn->_base.port);
      if (trusted_dirs_load_certs_from_string(body, 0, 1)<0) {
        log_warn(LD_DIR, "Unable to parse fetched certificates");
 +      /* if we fetched more than one and only some failed, the successful
 +       * ones got flushed to disk so it's safe to call this on them */
        connection_dir_download_cert_failed(conn, status_code);
      } else {
        directory_info_has_arrived(now, 0);
@@@ -1741,7 -1680,7 +1742,7 @@@
      const char *msg;
      int st;
      log_info(LD_DIR,"Got votes (size %d) from server %s:%d",
 -             (int) body_len, conn->_base.address, conn->_base.port);
 +             (int)body_len, conn->_base.address, conn->_base.port);
      if (status_code != 200) {
        log_warn(LD_DIR,
               "Received http status code %d (%s) from server "
@@@ -1761,11 -1700,11 +1762,11 @@@
    if (conn->_base.purpose == DIR_PURPOSE_FETCH_DETACHED_SIGNATURES) {
      const char *msg = NULL;
      log_info(LD_DIR,"Got detached signatures (size %d) from server %s:%d",
 -             (int) body_len, conn->_base.address, conn->_base.port);
 +             (int)body_len, conn->_base.address, conn->_base.port);
      if (status_code != 200) {
        log_warn(LD_DIR,
 -        "Received http status code %d (%s) from server "
 -        "'%s:%d' while fetching \"/tor/status-vote/consensus-signatures.z\".",
 +        "Received http status code %d (%s) from server '%s:%d' while fetching "
 +        "\"/tor/status-vote/next/consensus-signatures.z\".",
               status_code, escaped(reason), conn->_base.address,
               conn->_base.port);
        tor_free(body); tor_free(headers); tor_free(reason);
@@@ -1793,7 -1732,7 +1794,7 @@@
        which = smartlist_create();
        dir_split_resource_into_fingerprints(conn->requested_resource +
                                               (descriptor_digests ? 2 : 3),
 -                                           which, NULL, 0, 0);
 +                                           which, NULL, 0);
        n_asked_for = smartlist_len(which);
      }
      if (status_code != 200) {
@@@ -1981,7 -1920,7 +1982,7 @@@
            /* Success, or at least there's a v2 descriptor already
             * present. Notify pending connections about this. */
            conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC;
 -          rend_client_desc_trynow(conn->rend_data->onion_address, -1);
 +          rend_client_desc_trynow(conn->rend_data->onion_address);
          }
          break;
        case 404:
@@@ -2028,7 -1967,7 +2029,7 @@@
              log_info(LD_REND, "Successfully fetched v2 rendezvous "
                       "descriptor.");
              conn->_base.purpose = DIR_PURPOSE_HAS_FETCHED_RENDDESC;
 -            rend_client_desc_trynow(conn->rend_data->onion_address, -1);
 +            rend_client_desc_trynow(conn->rend_data->onion_address);
              break;
          }
          break;
@@@ -2071,6 -2010,12 +2072,6 @@@
                   "'%s:%d'. Malformed rendezvous descriptor?",
                   escaped(reason), conn->_base.address, conn->_base.port);
          break;
 -      case 503:
 -        log_info(LD_REND,"http status 503 (%s) response from dirserver "
 -                 "'%s:%d'. Node is (currently) not acting as v2 hidden "
 -                 "service directory.",
 -                 escaped(reason), conn->_base.address, conn->_base.port);
 -        break;
        default:
          log_warn(LD_REND,"http status %d (%s) response unexpected (server "
                   "'%s:%d').",
@@@ -2377,7 -2322,7 +2378,7 @@@ directory_dump_request_log(void
  }
  #endif
  
 -/** Decide whether a client would accept the consensus we have
 +/** Decide whether a client would accept the consensus we have.
   *
   * Clients can say they only want a consensus if it's signed by more
   * than half the authorities in a list.  They pass this list in
@@@ -2398,32 -2343,31 +2399,32 @@@ client_likes_consensus(networkstatus_t 
    int need_at_least;
    int have = 0;
  
 -  dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0, 0);
 +  dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
    need_at_least = smartlist_len(want_authorities)/2+1;
 -  SMARTLIST_FOREACH(want_authorities, const char *, d, {
 +  SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, d) {
      char want_digest[DIGEST_LEN];
      size_t want_len = strlen(d)/2;
      if (want_len > DIGEST_LEN)
        want_len = DIGEST_LEN;
  
      if (base16_decode(want_digest, DIGEST_LEN, d, want_len*2) < 0) {
 -      log_warn(LD_DIR,"Failed to decode requested authority digest %s.", d);
 +      log_fn(LOG_PROTOCOL_WARN, LD_DIR,
 +             "Failed to decode requested authority digest %s.", d);
        continue;
      };
  
 -    SMARTLIST_FOREACH(v->voters, networkstatus_voter_info_t *, vi, {
 -      if (vi->signature &&
 +    SMARTLIST_FOREACH_BEGIN(v->voters, networkstatus_voter_info_t *, vi) {
 +      if (smartlist_len(vi->sigs) &&
            !memcmp(vi->identity_digest, want_digest, want_len)) {
          have++;
          break;
        };
 -    });
 +    } SMARTLIST_FOREACH_END(vi);
  
      /* early exit, if we already have enough */
      if (have >= need_at_least)
        break;
 -  });
 +  } SMARTLIST_FOREACH_END(d);
  
    SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
    smartlist_free(want_authorities);
@@@ -2570,12 -2514,9 +2571,12 @@@ directory_handle_command_get(dir_connec
      /* v2 or v3 network status fetch. */
      smartlist_t *dir_fps = smartlist_create();
      int is_v3 = !strcmpstart(url, "/tor/status-vote");
 +    geoip_client_action_t act =
 +        is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2;
      const char *request_type = NULL;
      const char *key = url + strlen("/tor/status/");
      long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
 +
      if (!is_v3) {
        dirserv_get_networkstatus_v2_fingerprints(dir_fps, key);
        if (!strcmpstart(key, "fp/"))
@@@ -2590,44 -2531,18 +2591,44 @@@
      } else {
        networkstatus_t *v = networkstatus_get_latest_consensus();
        time_t now = time(NULL);
 +      const char *want_fps = NULL;
 +      char *flavor = NULL;
        #define CONSENSUS_URL_PREFIX "/tor/status-vote/current/consensus/"
 -      if (v &&
 -          !strcmpstart(url, CONSENSUS_URL_PREFIX) &&
 -          !client_likes_consensus(v, url + strlen(CONSENSUS_URL_PREFIX))) {
 +      #define CONSENSUS_FLAVORED_PREFIX "/tor/status-vote/current/consensus-"
 +      /* figure out the flavor if any, and who we wanted to sign the thing */
 +      if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
 +        const char *f, *cp;
 +        f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
 +        cp = strchr(f, '/');
 +        if (cp) {
 +          want_fps = cp+1;
 +          flavor = tor_strndup(f, cp-f);
 +        } else {
 +          flavor = tor_strdup(f);
 +        }
 +      } else {
 +        if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
 +          want_fps = url+strlen(CONSENSUS_URL_PREFIX);
 +      }
 +
 +      /* XXXX MICRODESC NM NM should check document of correct flavor */
 +      if (v && want_fps &&
 +          !client_likes_consensus(v, want_fps)) {
          write_http_status_line(conn, 404, "Consensus not signed by sufficient "
                                            "number of requested authorities");
          smartlist_free(dir_fps);
 +        geoip_note_ns_response(act, GEOIP_REJECT_NOT_ENOUGH_SIGS);
 +        tor_free(flavor);
          goto done;
        }
  
 -      smartlist_add(dir_fps, tor_memdup("\0\0\0\0\0\0\0\0\0\0"
 -                                        "\0\0\0\0\0\0\0\0\0\0", 20));
 +      {
 +        char *fp = tor_malloc_zero(DIGEST_LEN);
 +        if (flavor)
 +          strlcpy(fp, flavor, DIGEST_LEN);
 +        tor_free(flavor);
 +        smartlist_add(dir_fps, fp);
 +      }
        request_type = compressed?"v3.z":"v3";
        lifetime = (v && v->fresh_until > now) ? v->fresh_until - now : 0;
      }
@@@ -2635,7 -2550,6 +2636,7 @@@
      if (!smartlist_len(dir_fps)) { /* we failed to create/cache cp */
        write_http_status_line(conn, 503, "Network status object unavailable");
        smartlist_free(dir_fps);
 +      geoip_note_ns_response(act, GEOIP_REJECT_UNAVAILABLE);
        goto done;
      }
  
@@@ -2643,13 -2557,11 +2644,13 @@@
        write_http_status_line(conn, 404, "Not found");
        SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp));
        smartlist_free(dir_fps);
 +      geoip_note_ns_response(act, GEOIP_REJECT_NOT_FOUND);
        goto done;
      } else if (!smartlist_len(dir_fps)) {
        write_http_status_line(conn, 304, "Not modified");
        SMARTLIST_FOREACH(dir_fps, char *, cp, tor_free(cp));
        smartlist_free(dir_fps);
 +      geoip_note_ns_response(act, GEOIP_REJECT_NOT_MODIFIED);
        goto done;
      }
  
@@@ -2661,25 -2573,18 +2662,25 @@@
        write_http_status_line(conn, 503, "Directory busy, try again later");
        SMARTLIST_FOREACH(dir_fps, char *, fp, tor_free(fp));
        smartlist_free(dir_fps);
 +      geoip_note_ns_response(act, GEOIP_REJECT_BUSY);
        goto done;
      }
  
 -#ifdef ENABLE_GEOIP_STATS
      {
 -      geoip_client_action_t act =
 -        is_v3 ? GEOIP_CLIENT_NETWORKSTATUS : GEOIP_CLIENT_NETWORKSTATUS_V2;
        struct in_addr in;
 -      if (tor_inet_aton((TO_CONN(conn))->address, &in))
 +      if (tor_inet_aton((TO_CONN(conn))->address, &in)) {
          geoip_note_client_seen(act, ntohl(in.s_addr), time(NULL));
 +        geoip_note_ns_response(act, GEOIP_SUCCESS);
 +        /* Note that a request for a network status has started, so that we
 +         * can measure the download time later on. */
 +        if (TO_CONN(conn)->dirreq_id)
 +          geoip_start_dirreq(TO_CONN(conn)->dirreq_id, dlen, act,
 +                             DIRREQ_TUNNELED);
 +        else
 +          geoip_start_dirreq(TO_CONN(conn)->global_identifier, dlen, act,
 +                             DIRREQ_DIRECT);
 +      }
      }
 -#endif
  
      // note_request(request_type,dlen);
      (void) request_type;
@@@ -2715,7 -2620,7 +2716,7 @@@
        const char *item;
        tor_assert(!current); /* we handle current consensus specially above,
                               * since it wants to be spooled. */
 -      if ((item = dirvote_get_pending_consensus()))
 +      if ((item = dirvote_get_pending_consensus(FLAV_NS)))
          smartlist_add(items, (char*)item);
      } else if (!current && !strcmp(url, "consensus-signatures")) {
        /* XXXX the spec says that we should implement
@@@ -2741,8 -2646,7 +2742,8 @@@
          flags = DGV_BY_ID |
            (current ? DGV_INCLUDE_PREVIOUS : DGV_INCLUDE_PENDING);
        }
 -      dir_split_resource_into_fingerprints(url, fps, NULL, 1, 1);
 +      dir_split_resource_into_fingerprints(url, fps, NULL,
 +                                           DSR_HEX|DSR_SORT_UNIQ);
        SMARTLIST_FOREACH(fps, char *, fp, {
            if ((d = dirvote_get_vote(fp, flags)))
              smartlist_add(dir_items, (cached_dir_t*)d);
@@@ -2795,41 -2699,6 +2796,41 @@@
      goto done;
    }
  
 +  if (!strcmpstart(url, "/tor/micro/d/")) {
 +    smartlist_t *fps = smartlist_create();
 +
 +    dir_split_resource_into_fingerprints(url+strlen("/tor/micro/d/"),
 +                                      fps, NULL,
 +                                      DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
 +
 +    if (!dirserv_have_any_microdesc(fps)) {
 +      write_http_status_line(conn, 404, "Not found");
 +      SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp));
 +      smartlist_free(fps);
 +      goto done;
 +    }
 +    dlen = dirserv_estimate_microdesc_size(fps, compressed);
 +    if (global_write_bucket_low(TO_CONN(conn), dlen, 2)) {
 +      log_info(LD_DIRSERV,
 +               "Client asked for server descriptors, but we've been "
 +               "writing too many bytes lately. Sending 503 Dir busy.");
 +      write_http_status_line(conn, 503, "Directory busy, try again later");
 +      SMARTLIST_FOREACH(fps, char *, fp, tor_free(fp));
 +      smartlist_free(fps);
 +      goto done;
 +    }
 +
 +    write_http_response_header(conn, -1, compressed, MICRODESC_CACHE_LIFETIME);
 +    conn->dir_spool_src = DIR_SPOOL_MICRODESC;
 +    conn->fingerprint_stack = fps;
 +
 +    if (compressed)
 +      conn->zlib_state = tor_zlib_new(1, ZLIB_METHOD);
 +
 +    connection_dirserv_flushed_some(conn);
 +    goto done;
 +  }
 +
    if (!strcmpstart(url,"/tor/server/") ||
        (!options->BridgeAuthoritativeDir &&
         !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
@@@ -2911,8 -2780,7 +2912,8 @@@
      } else if (!strcmpstart(url, "/tor/keys/fp/")) {
        smartlist_t *fps = smartlist_create();
        dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"),
 -                                           fps, NULL, 1, 1);
 +                                           fps, NULL,
 +                                           DSR_HEX|DSR_SORT_UNIQ);
        SMARTLIST_FOREACH(fps, char *, d, {
            authority_cert_t *c = authority_cert_get_newest_by_id(d);
            if (c) smartlist_add(certs, c);
@@@ -2922,8 -2790,7 +2923,8 @@@
      } else if (!strcmpstart(url, "/tor/keys/sk/")) {
        smartlist_t *fps = smartlist_create();
        dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"),
 -                                           fps, NULL, 1, 1);
 +                                           fps, NULL,
 +                                           DSR_HEX|DSR_SORT_UNIQ);
        SMARTLIST_FOREACH(fps, char *, d, {
            authority_cert_t *c = authority_cert_get_by_sk_digest(d);
            if (c) smartlist_add(certs, c);
@@@ -3025,9 -2892,18 +3026,9 @@@
          note_request("/tor/rendezvous?/", desc_len);
          /* need to send descp separately, because it may include NULs */
          connection_write_to_buf(descp, desc_len, TO_CONN(conn));
          break;
        case 0: /* well-formed but not present */
          write_http_status_line(conn, 404, "Not found");
 -        /* report (unsuccessful) fetch to statistic */
 -        if (options->HSAuthorityRecordStats) {
 -          hs_usage_note_fetch_total(query, time(NULL));
 -        }
          break;
        case -1: /* not well-formed */
          write_http_status_line(conn, 400, "Bad request");
@@@ -3304,8 -3180,8 +3305,8 @@@ directory_handle_command(dir_connection
                                &body, &body_len, MAX_DIR_UL_SIZE, 0)) {
      case -1: /* overflow */
        log_warn(LD_DIRSERV,
 -               "Invalid input from address '%s'. Closing.",
 -               conn->_base.address);
 +               "Request too large from address '%s' to DirPort. Closing.",
 +               safe_str(conn->_base.address));
        return -1;
      case 0:
        log_debug(LD_DIRSERV,"command not all here yet.");
@@@ -3341,16 -3217,6 +3342,16 @@@ connection_dir_finished_flushing(dir_co
    tor_assert(conn);
    tor_assert(conn->_base.type == CONN_TYPE_DIR);
  
 +  /* Note that we have finished writing the directory response. For direct
 +   * connections this means we're done, for tunneled connections its only
 +   * an intermediate step. */
 +  if (TO_CONN(conn)->dirreq_id)
 +    geoip_change_dirreq_state(TO_CONN(conn)->dirreq_id, DIRREQ_TUNNELED,
 +                              DIRREQ_FLUSHING_DIR_CONN_FINISHED);
 +  else
 +    geoip_change_dirreq_state(TO_CONN(conn)->global_identifier,
 +                              DIRREQ_DIRECT,
 +                              DIRREQ_FLUSHING_DIR_CONN_FINISHED);
    switch (conn->_base.state) {
      case DIR_CONN_STATE_CLIENT_SENDING:
        log_debug(LD_DIR,"client finished sending command.");
@@@ -3533,14 -3399,6 +3534,14 @@@ download_status_reset(download_status_
    dls->next_attempt_at = time(NULL) + schedule[0];
  }
  
 +/** Return the number of failures on <b>dls</b> since the last success (if
 + * any). */
 +int
 +download_status_get_n_failures(const download_status_t *dls)
 +{
 +  return dls->n_download_failures;
 +}
 +
  /** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>)
   * fetches have failed (with uppercase fingerprints listed in <b>failed</b>,
   * either as descriptor digests or as identity digests based on
@@@ -3556,8 -3414,16 +3557,8 @@@ dir_routerdesc_download_failed(smartlis
    int server = directory_fetches_from_authorities(get_options());
    if (!was_descriptor_digests) {
      if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
 -      tor_assert(!was_extrainfo); /* not supported yet */
 -      SMARTLIST_FOREACH(failed, const char *, cp,
 -      {
 -        if (base16_decode(digest, DIGEST_LEN, cp, strlen(cp))<0) {
 -          log_warn(LD_BUG, "Malformed fingerprint in list: %s",
 -                   escaped(cp));
 -          continue;
 -        }
 -        retry_bridge_descriptor_fetch_directly(digest);
 -      });
 +      tor_assert(!was_extrainfo);
 +      connection_dir_retry_bridges(failed);
      }
      return; /* FFFF should implement for other-than-router-purpose someday */
    }
@@@ -3650,37 -3516,19 +3651,37 @@@ dir_split_resource_into_fingerprint_pai
  /** Given a directory <b>resource</b> request, containing zero
   * or more strings separated by plus signs, followed optionally by ".z", store
   * the strings, in order, into <b>fp_out</b>.  If <b>compressed_out</b> is
 - * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.  If
 - * decode_hex is true, then delete all elements that aren't hex digests, and
 - * decode the rest.  If sort_uniq is true, then sort the list and remove
 - * all duplicates.
 + * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.
 + *
 + * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and
 + * decode the rest.  If (flags & DSR_BASE64), then use "-" rather than "+" as
 + * a separator, delete all the elements that aren't base64-encoded digests,
 + * and decode the rest.  If (flags & DSR_DIGEST256), these digests should be
 + * 256 bits long; else they should be 160.
 + *
 + * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates.
   */
  int
  dir_split_resource_into_fingerprints(const char *resource,
                                       smartlist_t *fp_out, int *compressed_out,
 -                                     int decode_hex, int sort_uniq)
 +                                     int flags)
  {
 +  const int decode_hex = flags & DSR_HEX;
 +  const int decode_base64 = flags & DSR_BASE64;
 +  const int digests_are_256 = flags & DSR_DIGEST256;
 +  const int sort_uniq = flags & DSR_SORT_UNIQ;
 +
 +  const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN;
 +  const int hex_digest_len = digests_are_256 ?
 +    HEX_DIGEST256_LEN : HEX_DIGEST_LEN;
 +  const int base64_digest_len = digests_are_256 ?
 +    BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN;
    smartlist_t *fp_tmp = smartlist_create();
 +
 +  tor_assert(!(decode_hex && decode_base64));
    tor_assert(fp_out);
 -  smartlist_split_string(fp_tmp, resource, "+", 0, 0);
 +
 +  smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0);
    if (compressed_out)
      *compressed_out = 0;
    if (smartlist_len(fp_tmp)) {
@@@ -3692,25 -3540,22 +3693,25 @@@
          *compressed_out = 1;
      }
    }
 -  if (decode_hex) {
 +  if (decode_hex || decode_base64) {
 +    const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len;
      int i;
      char *cp, *d = NULL;
      for (i = 0; i < smartlist_len(fp_tmp); ++i) {
        cp = smartlist_get(fp_tmp, i);
 -      if (strlen(cp) != HEX_DIGEST_LEN) {
 +      if (strlen(cp) != encoded_len) {
          log_info(LD_DIR,
                   "Skipping digest %s with non-standard length.", escaped(cp));
          smartlist_del_keeporder(fp_tmp, i--);
          goto again;
        }
 -      d = tor_malloc_zero(DIGEST_LEN);
 -      if (base16_decode(d, DIGEST_LEN, cp, HEX_DIGEST_LEN)<0) {
 -        log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp));
 -        smartlist_del_keeporder(fp_tmp, i--);
 -        goto again;
 +      d = tor_malloc_zero(digest_len);
 +      if (decode_hex ?
 +          (base16_decode(d, digest_len, cp, hex_digest_len)<0) :
 +          (base64_decode(d, digest_len, cp, base64_digest_len)<0)) {
 +          log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp));
 +          smartlist_del_keeporder(fp_tmp, i--);
 +          goto again;
        }
        smartlist_set(fp_tmp, i, d);
        d = NULL;
@@@ -3720,18 -3565,26 +3721,18 @@@
      }
    }
    if (sort_uniq) {
 -    smartlist_t *fp_tmp2 = smartlist_create();
 -    int i;
 -    if (decode_hex)
 -      smartlist_sort_digests(fp_tmp);
 -    else
 +    if (decode_hex || decode_base64) {
 +      if (digests_are_256) {
 +        smartlist_sort_digests256(fp_tmp);
 +        smartlist_uniq_digests256(fp_tmp);
 +      } else {
 +        smartlist_sort_digests(fp_tmp);
 +        smartlist_uniq_digests(fp_tmp);
 +      }
 +    } else {
        smartlist_sort_strings(fp_tmp);
 -    if (smartlist_len(fp_tmp))
 -      smartlist_add(fp_tmp2, smartlist_get(fp_tmp, 0));
 -    for (i = 1; i < smartlist_len(fp_tmp); ++i) {
 -      char *cp = smartlist_get(fp_tmp, i);
 -      char *last = smartlist_get(fp_tmp2, smartlist_len(fp_tmp2)-1);
 -
 -      if ((decode_hex && memcmp(cp, last, DIGEST_LEN))
 -          || (!decode_hex && strcasecmp(cp, last)))
 -        smartlist_add(fp_tmp2, cp);
 -      else
 -        tor_free(cp);
 +      smartlist_uniq_strings(fp_tmp);
      }
 -    smartlist_free(fp_tmp);
 -    fp_tmp = fp_tmp2;
    }
    smartlist_add_all(fp_out, fp_tmp);
    smartlist_free(fp_tmp);