[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [tor/master] Split directory.c code into several modules
commit 194acfb51d9b26ae63f9d48d3442c4ff64cbc8cc
Author: Nick Mathewson <nickm@xxxxxxxxxxxxxx>
Date: Fri Sep 21 10:26:57 2018 -0400
Split directory.c code into several modules
Parts of this C file naturally belong in dircache, dirclient, and
dircommon: so, move them there.
---
changes/ticket26744 | 3 +
src/core/include.am | 10 +-
src/core/mainloop/connection.c | 2 +-
src/core/mainloop/mainloop.c | 2 +-
src/core/or/circuitbuild.c | 2 +-
src/core/or/circuitlist.c | 2 +-
src/core/or/circuituse.c | 2 +-
src/core/or/connection_edge.c | 2 +-
src/core/or/relay.c | 2 +-
src/feature/client/bridges.c | 4 +-
src/feature/client/entrynodes.c | 2 +-
src/feature/control/control.c | 4 +-
src/feature/dirauth/dirvote.c | 3 +-
src/feature/dirauth/process_descs.c | 3 +-
src/feature/dircache/dircache.c | 1739 ++++++++++++
src/feature/dircache/dircache.h | 43 +
src/feature/dircache/directory.h | 347 ---
src/feature/dircache/dirserv.c | 2 +-
.../directory.c => dirclient/dirclient.c} | 2961 +-------------------
src/feature/dirclient/dirclient.h | 172 ++
src/feature/dirclient/dlstatus.c | 422 +++
src/feature/dirclient/dlstatus.h | 58 +
src/feature/dircommon/directory.c | 651 +++++
src/feature/dircommon/directory.h | 129 +
src/feature/hs/hs_client.c | 3 +-
src/feature/hs/hs_service.c | 3 +-
src/feature/nodelist/authcert.c | 4 +-
src/feature/nodelist/dirlist.c | 2 +-
src/feature/nodelist/microdesc.c | 3 +-
src/feature/nodelist/networkstatus.c | 4 +-
src/feature/nodelist/node_select.c | 3 +-
src/feature/nodelist/routerlist.c | 4 +-
src/feature/relay/router.c | 3 +-
src/feature/rend/rendclient.c | 3 +-
src/feature/rend/rendservice.c | 3 +-
src/test/fuzz/fuzz_http.c | 5 +-
src/test/test_connection.c | 2 +-
src/test/test_dir.c | 8 +-
src/test/test_dir_handle_get.c | 4 +-
src/test/test_entrynodes.c | 5 +-
src/test/test_hs_cache.c | 6 +-
src/test/test_hs_client.c | 2 +-
src/test/test_hs_common.c | 2 +-
src/test/test_oos.c | 2 +-
src/test/test_routerlist.c | 5 +-
45 files changed, 3390 insertions(+), 3253 deletions(-)
diff --git a/changes/ticket26744 b/changes/ticket26744
new file mode 100644
index 000000000..10ff51f06
--- /dev/null
+++ b/changes/ticket26744
@@ -0,0 +1,3 @@
+ o Code simplification and refactoring:
+ - Split directory.c into separate pieces for client, server, and
+ common functionality. Closes ticket 26744.
diff --git a/src/core/include.am b/src/core/include.am
index d042f697b..61056d7fd 100644
--- a/src/core/include.am
+++ b/src/core/include.am
@@ -62,9 +62,12 @@ LIBTOR_APP_A_SOURCES = \
src/feature/dirauth/keypin.c \
src/feature/dircache/conscache.c \
src/feature/dircache/consdiffmgr.c \
- src/feature/dircache/directory.c \
+ src/feature/dircache/dircache.c \
src/feature/dircache/dirserv.c \
+ src/feature/dirclient/dirclient.c \
+ src/feature/dirclient/dlstatus.c \
src/feature/dircommon/consdiff.c \
+ src/feature/dircommon/directory.c \
src/feature/dircommon/fp_pair.c \
src/feature/dircommon/voting_schedule.c \
src/feature/hibernate/hibernate.c \
@@ -257,12 +260,15 @@ noinst_HEADERS += \
src/feature/dircache/cached_dir_st.h \
src/feature/dircache/conscache.h \
src/feature/dircache/consdiffmgr.h \
- src/feature/dircache/directory.h \
+ src/feature/dircache/dircache.h \
src/feature/dircache/dirserv.h \
src/feature/dirclient/dir_server_st.h \
+ src/feature/dirclient/dirclient.h \
+ src/feature/dirclient/dlstatus.h \
src/feature/dirclient/download_status_st.h \
src/feature/dircommon/consdiff.h \
src/feature/dircommon/dir_connection_st.h \
+ src/feature/dircommon/directory.h \
src/feature/dircommon/fp_pair.h \
src/feature/dircommon/vote_timing_st.h \
src/feature/dircommon/voting_schedule.h \
diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c
index f85a2b61d..4231bec01 100644
--- a/src/core/mainloop/connection.c
+++ b/src/core/mainloop/connection.c
@@ -77,7 +77,7 @@
#include "core/or/connection_or.h"
#include "feature/control/control.h"
#include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "feature/dircache/dirserv.h"
#include "feature/relay/dns.h"
#include "feature/client/dnsserv.h"
diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c
index 4ab00f92d..1072a23b7 100644
--- a/src/core/mainloop/mainloop.c
+++ b/src/core/mainloop/mainloop.c
@@ -68,7 +68,7 @@
#include "feature/control/control.h"
#include "core/mainloop/cpuworker.h"
#include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "feature/dircache/dirserv.h"
#include "feature/dirauth/reachability.h"
#include "feature/relay/dns.h"
diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c
index 74f60e6c4..2f404d179 100644
--- a/src/core/or/circuitbuild.c
+++ b/src/core/or/circuitbuild.c
@@ -44,7 +44,7 @@
#include "core/or/connection_or.h"
#include "feature/control/control.h"
#include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "feature/client/entrynodes.h"
#include "core/crypto/hs_ntor.h"
#include "core/mainloop/mainloop.h"
diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c
index f231beb61..c07c8e97a 100644
--- a/src/core/or/circuitlist.c
+++ b/src/core/or/circuitlist.c
@@ -69,7 +69,7 @@
#include "lib/crypt_ops/crypto_rand.h"
#include "lib/crypt_ops/crypto_util.h"
#include "lib/crypt_ops/crypto_dh.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "feature/client/entrynodes.h"
#include "core/mainloop/mainloop.h"
#include "feature/hs/hs_circuit.h"
diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c
index 0f2b1ede3..7b1f31559 100644
--- a/src/core/or/circuituse.c
+++ b/src/core/or/circuituse.c
@@ -40,7 +40,7 @@
#include "core/mainloop/connection.h"
#include "core/or/connection_edge.h"
#include "feature/control/control.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "feature/client/entrynodes.h"
#include "feature/hs/hs_common.h"
#include "feature/hs/hs_client.h"
diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c
index d0fad6700..daf978b99 100644
--- a/src/core/or/connection_edge.c
+++ b/src/core/or/connection_edge.c
@@ -73,7 +73,7 @@
#include "lib/crypt_ops/crypto_util.h"
#include "feature/relay/dns.h"
#include "feature/client/dnsserv.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "feature/dircache/dirserv.h"
#include "feature/hibernate/hibernate.h"
#include "feature/hs/hs_common.h"
diff --git a/src/core/or/relay.c b/src/core/or/relay.c
index 4638d9f21..47be0f403 100644
--- a/src/core/or/relay.c
+++ b/src/core/or/relay.c
@@ -63,7 +63,7 @@
#include "feature/control/control.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "feature/relay/dns.h"
#include "feature/stats/geoip.h"
#include "feature/hs/hs_cache.h"
diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c
index 5ab0f6f4f..e8afb5a92 100644
--- a/src/feature/client/bridges.c
+++ b/src/feature/client/bridges.c
@@ -18,7 +18,9 @@
#include "core/or/circuitbuild.h"
#include "app/config/config.h"
#include "core/mainloop/connection.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
#include "feature/client/entrynodes.h"
#include "feature/nodelist/nodelist.h"
#include "core/or/policies.h"
diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c
index 94bb011fb..521bf33b5 100644
--- a/src/feature/client/entrynodes.c
+++ b/src/feature/client/entrynodes.c
@@ -125,7 +125,7 @@
#include "core/mainloop/connection.h"
#include "feature/control/control.h"
#include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "feature/client/entrynodes.h"
#include "core/mainloop/mainloop.h"
#include "feature/nodelist/microdesc.h"
diff --git a/src/feature/control/control.c b/src/feature/control/control.c
index b5a0dfeaf..f307101ed 100644
--- a/src/feature/control/control.c
+++ b/src/feature/control/control.c
@@ -56,7 +56,9 @@
#include "feature/control/fmt_serverstatus.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
#include "feature/dircache/dirserv.h"
#include "feature/client/dnsserv.h"
#include "feature/client/entrynodes.h"
diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c
index 5ca9ed4a4..a1fe0d423 100644
--- a/src/feature/dirauth/dirvote.c
+++ b/src/feature/dirauth/dirvote.c
@@ -9,7 +9,8 @@
#include "feature/dirauth/dircollate.h"
#include "feature/dirauth/recommend_pkg.h"
#include "feature/dirauth/voteflags.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
#include "feature/dirauth/bwauth.h"
#include "feature/dircache/dirserv.h"
#include "feature/dirauth/guardfraction.h"
diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c
index 2c2fefca7..1cf31f85c 100644
--- a/src/feature/dirauth/process_descs.c
+++ b/src/feature/dirauth/process_descs.c
@@ -19,7 +19,8 @@
#include "core/or/policies.h"
#include "feature/dirauth/keypin.h"
#include "feature/dirauth/reachability.h"
-#include "feature/dircache/directory.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/directory.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/nodelist.h"
#include "feature/nodelist/routerlist.h"
diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c
new file mode 100644
index 000000000..ef4fe391b
--- /dev/null
+++ b/src/feature/dircache/dircache.c
@@ -0,0 +1,1739 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DIRCACHE_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/relay.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/mode.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/dircache/conscache.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dircache.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircommon/fp_pair.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/rend/rendcache.h"
+#include "feature/stats/geoip.h"
+#include "feature/stats/rephist.h"
+#include "lib/compress/compress.h"
+
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/authority_cert_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+/** Maximum size, in bytes, for any directory object that we're accepting
+ * as an upload. */
+#define MAX_DIR_UL_SIZE ((1<<24)-1) /* 16MB-1 */
+
+/** HTTP cache control: how long do we tell proxies they can cache each
+ * kind of document we serve? */
+#define FULL_DIR_CACHE_LIFETIME (60*60)
+#define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
+#define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
+#define NETWORKSTATUS_CACHE_LIFETIME (5*60)
+#define ROUTERDESC_CACHE_LIFETIME (30*60)
+#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
+#define ROBOTS_CACHE_LIFETIME (24*60*60)
+#define MICRODESC_CACHE_LIFETIME (48*60*60)
+
+/** Parse an HTTP request string <b>headers</b> of the form
+ * \verbatim
+ * "\%s [http[s]://]\%s HTTP/1..."
+ * \endverbatim
+ * If it's well-formed, strdup the second \%s into *<b>url</b>, and
+ * nul-terminate it. If the url doesn't start with "/tor/", rewrite it
+ * so it does. Return 0.
+ * Otherwise, return -1.
+ */
+STATIC int
+parse_http_url(const char *headers, char **url)
+{
+ char *command = NULL;
+ if (parse_http_command(headers, &command, url) < 0) {
+ return -1;
+ }
+ if (strcmpstart(*url, "/tor/")) {
+ char *new_url = NULL;
+ tor_asprintf(&new_url, "/tor%s%s",
+ *url[0] == '/' ? "" : "/",
+ *url);
+ tor_free(*url);
+ *url = new_url;
+ }
+ tor_free(command);
+ return 0;
+}
+
+/** Create an http response for the client <b>conn</b> out of
+ * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>.
+ */
+static void
+write_short_http_response(dir_connection_t *conn, int status,
+ const char *reason_phrase)
+{
+ char *buf = NULL;
+ char *datestring = NULL;
+
+ IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */
+ reason_phrase = "unspecified";
+ }
+
+ if (server_mode(get_options())) {
+ /* include the Date: header, but only if we're a relay or bridge */
+ char datebuf[RFC1123_TIME_LEN+1];
+ format_rfc1123_time(datebuf, time(NULL));
+ tor_asprintf(&datestring, "Date: %s\r\n", datebuf);
+ }
+
+ tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n",
+ status, reason_phrase, datestring?datestring:"");
+
+ log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
+ connection_buf_add(buf, strlen(buf), TO_CONN(conn));
+
+ tor_free(datestring);
+ tor_free(buf);
+}
+
+/** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
+ * with <b>type</b> as the Content-Type.
+ *
+ * If <b>length</b> is nonnegative, it is the Content-Length.
+ * If <b>encoding</b> is provided, it is the Content-Encoding.
+ * If <b>cache_lifetime</b> is greater than 0, the content may be cached for
+ * up to cache_lifetime seconds. Otherwise, the content may not be cached. */
+static void
+write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
+ const char *type, const char *encoding,
+ const char *extra_headers,
+ long cache_lifetime)
+{
+ char date[RFC1123_TIME_LEN+1];
+ time_t now = time(NULL);
+ buf_t *buf = buf_new_with_capacity(1024);
+
+ tor_assert(conn);
+
+ format_rfc1123_time(date, now);
+
+ buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date);
+ if (type) {
+ buf_add_printf(buf, "Content-Type: %s\r\n", type);
+ }
+ if (!is_local_addr(&conn->base_.addr)) {
+ /* Don't report the source address for a nearby/private connection.
+ * Otherwise we tend to mis-report in cases where incoming ports are
+ * being forwarded to a Tor server running behind the firewall. */
+ buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address);
+ }
+ if (encoding) {
+ buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding);
+ }
+ if (length >= 0) {
+ buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length);
+ }
+ if (cache_lifetime > 0) {
+ char expbuf[RFC1123_TIME_LEN+1];
+ format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime));
+ /* We could say 'Cache-control: max-age=%d' here if we start doing
+ * http/1.1 */
+ buf_add_printf(buf, "Expires: %s\r\n", expbuf);
+ } else if (cache_lifetime == 0) {
+ /* We could say 'Cache-control: no-cache' here if we start doing
+ * http/1.1 */
+ buf_add_string(buf, "Pragma: no-cache\r\n");
+ }
+ if (extra_headers) {
+ buf_add_string(buf, extra_headers);
+ }
+ buf_add_string(buf, "\r\n");
+
+ connection_buf_add_buf(TO_CONN(conn), buf);
+ buf_free(buf);
+}
+
+/** As write_http_response_header_impl, but sets encoding and content-typed
+ * based on whether the response will be <b>compressed</b> or not. */
+static void
+write_http_response_headers(dir_connection_t *conn, ssize_t length,
+ compress_method_t method,
+ const char *extra_headers, long cache_lifetime)
+{
+ const char *methodname = compression_method_get_name(method);
+ const char *doctype;
+ if (method == NO_METHOD)
+ doctype = "text/plain";
+ else
+ doctype = "application/octet-stream";
+ write_http_response_header_impl(conn, length,
+ doctype,
+ methodname,
+ extra_headers,
+ cache_lifetime);
+}
+
+/** As write_http_response_headers, but assumes extra_headers is NULL */
+static void
+write_http_response_header(dir_connection_t *conn, ssize_t length,
+ compress_method_t method,
+ long cache_lifetime)
+{
+ write_http_response_headers(conn, length, method, NULL, cache_lifetime);
+}
+
+/** Array of compression methods to use (if supported) for serving
+ * precompressed data, ordered from best to worst. */
+static compress_method_t srv_meth_pref_precompressed[] = {
+ LZMA_METHOD,
+ ZSTD_METHOD,
+ ZLIB_METHOD,
+ GZIP_METHOD,
+ NO_METHOD
+};
+
+/** Array of compression methods to use (if supported) for serving
+ * streamed data, ordered from best to worst. */
+static compress_method_t srv_meth_pref_streaming_compression[] = {
+ ZSTD_METHOD,
+ ZLIB_METHOD,
+ GZIP_METHOD,
+ NO_METHOD
+};
+
+/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>,
+ * and convert them to a bitfield where compression method x is supported if
+ * and only if 1 << x is set in the bitfield. */
+STATIC unsigned
+parse_accept_encoding_header(const char *h)
+{
+ unsigned result = (1u << NO_METHOD);
+ smartlist_t *methods = smartlist_new();
+ smartlist_split_string(methods, h, ",",
+ SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+ SMARTLIST_FOREACH_BEGIN(methods, const char *, m) {
+ compress_method_t method = compression_method_get_by_name(m);
+ if (method != UNKNOWN_METHOD) {
+ tor_assert(((unsigned)method) < 8*sizeof(unsigned));
+ result |= (1u << method);
+ }
+ } SMARTLIST_FOREACH_END(m);
+ SMARTLIST_FOREACH_BEGIN(methods, char *, m) {
+ tor_free(m);
+ } SMARTLIST_FOREACH_END(m);
+ smartlist_free(methods);
+ return result;
+}
+
+/** Decide whether a client would accept the consensus we have.
+ *
+ * Clients can say they only want a consensus if it's signed by more
+ * than half the authorities in a list. They pass this list in
+ * the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>".
+ *
+ * <b>fpr</b> may be an abbreviated fingerprint, i.e. only a left substring
+ * of the full authority identity digest. (Only strings of even length,
+ * i.e. encodings of full bytes, are handled correctly. In the case
+ * of an odd number of hex digits the last one is silently ignored.)
+ *
+ * Returns 1 if more than half of the requested authorities signed the
+ * consensus, 0 otherwise.
+ */
+static int
+client_likes_consensus(const struct consensus_cache_entry_t *ent,
+ const char *want_url)
+{
+ smartlist_t *voters = smartlist_new();
+ int need_at_least;
+ int have = 0;
+
+ if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) {
+ smartlist_free(voters);
+ return 1; // We don't know the voters; assume the client won't mind. */
+ }
+
+ smartlist_t *want_authorities = smartlist_new();
+ dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
+ need_at_least = smartlist_len(want_authorities)/2+1;
+
+ SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) {
+
+ SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) {
+ if (!strcasecmpstart(digest, want_digest)) {
+ have++;
+ break;
+ };
+ } SMARTLIST_FOREACH_END(digest);
+
+ /* early exit, if we already have enough */
+ if (have >= need_at_least)
+ break;
+ } SMARTLIST_FOREACH_END(want_digest);
+
+ SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
+ smartlist_free(want_authorities);
+ SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp));
+ smartlist_free(voters);
+ return (have >= need_at_least);
+}
+
+/** Return the compression level we should use for sending a compressed
+ * response of size <b>n_bytes</b>. */
+STATIC compression_level_t
+choose_compression_level(ssize_t n_bytes)
+{
+ if (! have_been_under_memory_pressure()) {
+ return HIGH_COMPRESSION; /* we have plenty of RAM. */
+ } else if (n_bytes < 0) {
+ return HIGH_COMPRESSION; /* unknown; might be big. */
+ } else if (n_bytes < 1024) {
+ return LOW_COMPRESSION;
+ } else if (n_bytes < 2048) {
+ return MEDIUM_COMPRESSION;
+ } else {
+ return HIGH_COMPRESSION;
+ }
+}
+
+/** Information passed to handle a GET request. */
+typedef struct get_handler_args_t {
+ /** Bitmask of compression methods that the client said (or implied) it
+ * supported. */
+ unsigned compression_supported;
+ /** If nonzero, the time included an if-modified-since header with this
+ * value. */
+ time_t if_modified_since;
+ /** String containing the requested URL or resource. */
+ const char *url;
+ /** String containing the HTTP headers */
+ const char *headers;
+} get_handler_args_t;
+
+/** Entry for handling an HTTP GET request.
+ *
+ * This entry matches a request if "string" is equal to the requested
+ * resource, or if "is_prefix" is true and "string" is a prefix of the
+ * requested resource.
+ *
+ * The 'handler' function is called to handle the request. It receives
+ * an arguments structure, and must return 0 on success or -1 if we should
+ * close the connection.
+ **/
+typedef struct url_table_ent_s {
+ const char *string;
+ int is_prefix;
+ int (*handler)(dir_connection_t *conn, const get_handler_args_t *args);
+} url_table_ent_t;
+
+static int handle_get_frontpage(dir_connection_t *conn,
+ const get_handler_args_t *args);
+static int handle_get_current_consensus(dir_connection_t *conn,
+ const get_handler_args_t *args);
+static int handle_get_status_vote(dir_connection_t *conn,
+ const get_handler_args_t *args);
+static int handle_get_microdesc(dir_connection_t *conn,
+ const get_handler_args_t *args);
+static int handle_get_descriptor(dir_connection_t *conn,
+ const get_handler_args_t *args);
+static int handle_get_keys(dir_connection_t *conn,
+ const get_handler_args_t *args);
+static int handle_get_hs_descriptor_v2(dir_connection_t *conn,
+ const get_handler_args_t *args);
+static int handle_get_robots(dir_connection_t *conn,
+ const get_handler_args_t *args);
+static int handle_get_networkstatus_bridges(dir_connection_t *conn,
+ const get_handler_args_t *args);
+
+/** Table for handling GET requests. */
+static const url_table_ent_t url_table[] = {
+ { "/tor/", 0, handle_get_frontpage },
+ { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
+ { "/tor/status-vote/current/", 1, handle_get_status_vote },
+ { "/tor/status-vote/next/", 1, handle_get_status_vote },
+ { "/tor/micro/d/", 1, handle_get_microdesc },
+ { "/tor/server/", 1, handle_get_descriptor },
+ { "/tor/extra/", 1, handle_get_descriptor },
+ { "/tor/keys/", 1, handle_get_keys },
+ { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 },
+ { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
+ { "/tor/robots.txt", 0, handle_get_robots },
+ { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
+ { NULL, 0, NULL },
+};
+
+/** Helper function: called when a dirserver gets a complete HTTP GET
+ * request. Look for a request for a directory or for a rendezvous
+ * service descriptor. On finding one, write a response into
+ * conn-\>outbuf. If the request is unrecognized, send a 404.
+ * Return 0 if we handled this successfully, or -1 if we need to close
+ * the connection. */
+MOCK_IMPL(STATIC int,
+directory_handle_command_get,(dir_connection_t *conn, const char *headers,
+ const char *req_body, size_t req_body_len))
+{
+ char *url, *url_mem, *header;
+ time_t if_modified_since = 0;
+ int zlib_compressed_in_url;
+ unsigned compression_methods_supported;
+
+ /* We ignore the body of a GET request. */
+ (void)req_body;
+ (void)req_body_len;
+
+ log_debug(LD_DIRSERV,"Received GET command.");
+
+ conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
+
+ if (parse_http_url(headers, &url) < 0) {
+ write_short_http_response(conn, 400, "Bad request");
+ return 0;
+ }
+ if ((header = http_get_header(headers, "If-Modified-Since: "))) {
+ struct tm tm;
+ if (parse_http_time(header, &tm) == 0) {
+ if (tor_timegm(&tm, &if_modified_since)<0) {
+ if_modified_since = 0;
+ } else {
+ log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header));
+ }
+ }
+ /* The correct behavior on a malformed If-Modified-Since header is to
+ * act as if no If-Modified-Since header had been given. */
+ tor_free(header);
+ }
+ log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
+
+ url_mem = url;
+ {
+ size_t url_len = strlen(url);
+
+ zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z");
+ if (zlib_compressed_in_url) {
+ url[url_len-2] = '\0';
+ }
+ }
+
+ if ((header = http_get_header(headers, "Accept-Encoding: "))) {
+ compression_methods_supported = parse_accept_encoding_header(header);
+ tor_free(header);
+ } else {
+ compression_methods_supported = (1u << NO_METHOD);
+ }
+ if (zlib_compressed_in_url) {
+ compression_methods_supported |= (1u << ZLIB_METHOD);
+ }
+
+ /* Remove all methods that we don't both support. */
+ compression_methods_supported &= tor_compress_get_supported_method_bitmask();
+
+ get_handler_args_t args;
+ args.url = url;
+ args.headers = headers;
+ args.if_modified_since = if_modified_since;
+ args.compression_supported = compression_methods_supported;
+
+ int i, result = -1;
+ for (i = 0; url_table[i].string; ++i) {
+ int match;
+ if (url_table[i].is_prefix) {
+ match = !strcmpstart(url, url_table[i].string);
+ } else {
+ match = !strcmp(url, url_table[i].string);
+ }
+ if (match) {
+ result = url_table[i].handler(conn, &args);
+ goto done;
+ }
+ }
+
+ /* we didn't recognize the url */
+ write_short_http_response(conn, 404, "Not found");
+ result = 0;
+
+ done:
+ tor_free(url_mem);
+ return result;
+}
+
+/** Helper function for GET / or GET /tor/
+ */
+static int
+handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
+{
+ (void) args; /* unused */
+ const char *frontpage = get_dirportfrontpage();
+
+ if (frontpage) {
+ size_t dlen;
+ dlen = strlen(frontpage);
+ /* Let's return a disclaimer page (users shouldn't use V1 anymore,
+ and caches don't fetch '/', so this is safe). */
+
+ /* [We don't check for write_bucket_low here, since we want to serve
+ * this page no matter what.] */
+ write_http_response_header_impl(conn, dlen, "text/html", "identity",
+ NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
+ connection_buf_add(frontpage, dlen, TO_CONN(conn));
+ } else {
+ write_short_http_response(conn, 404, "Not found");
+ }
+ return 0;
+}
+
+/** Warn that the cached consensus <b>consensus</b> of type
+ * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
+ * warning to avoid logging an entry on every request.
+ */
+static void
+warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
+ const char *flavor, time_t now)
+{
+#define TOO_OLD_WARNING_INTERVAL (60*60)
+ static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
+ char timestamp[ISO_TIME_LEN+1];
+ time_t valid_until;
+ char *dupes;
+
+ if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
+ return;
+
+ if ((dupes = rate_limit_log(&warned, now))) {
+ format_local_iso_time(timestamp, valid_until);
+ log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
+ "serve it to clients. It was valid until %s local time and we "
+ "continued to serve it for up to 24 hours after it expired.%s",
+ flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
+ tor_free(dupes);
+ }
+}
+
+/**
+ * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into
+ * <b>digest</b>. Return 0 on success. On failure, report that the hash came
+ * from <b>location</b>, report that we are taking <b>action</b> with it, and
+ * return -1.
+ */
+static int
+parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
+ const char *action)
+{
+ if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) ==
+ DIGEST256_LEN) {
+ return 0;
+ } else {
+ log_fn(LOG_PROTOCOL_WARN, LD_DIR,
+ "%s contained bogus digest %s; %s.",
+ location, escaped(hex), action);
+ return -1;
+ }
+}
+
+/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>,
+ * set <b>digest_out<b> to a new smartlist containing every 256-bit
+ * hex-encoded digest listed in that header and return 0. Otherwise return
+ * -1. */
+static int
+parse_or_diff_from_header(smartlist_t **digests_out, const char *headers)
+{
+ char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER);
+ if (hdr == NULL) {
+ return -1;
+ }
+ smartlist_t *hex_digests = smartlist_new();
+ *digests_out = smartlist_new();
+ smartlist_split_string(hex_digests, hdr, " ",
+ SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+ SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) {
+ uint8_t digest[DIGEST256_LEN];
+ if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header",
+ "ignoring")) {
+ smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest)));
+ }
+ } SMARTLIST_FOREACH_END(hex);
+ SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp));
+ smartlist_free(hex_digests);
+ tor_free(hdr);
+ return 0;
+}
+
+/** Fallback compression method. The fallback compression method is used in
+ * case a client requests a non-compressed document. We only store compressed
+ * documents, so we use this compression method to fetch the document and let
+ * the spooling system do the streaming decompression.
+ */
+#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD
+
+/**
+ * Try to find the best consensus diff possible in order to serve a client
+ * request for a diff from one of the consensuses in <b>digests</b> to the
+ * current consensus of flavor <b>flav</b>. The client supports the
+ * compression methods listed in the <b>compression_methods</b> bitfield:
+ * place the method chosen (if any) into <b>compression_used_out</b>.
+ */
+static struct consensus_cache_entry_t *
+find_best_diff(const smartlist_t *digests, int flav,
+ unsigned compression_methods,
+ compress_method_t *compression_used_out)
+{
+ struct consensus_cache_entry_t *result = NULL;
+
+ SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
+ unsigned u;
+ for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
+ compress_method_t method = srv_meth_pref_precompressed[u];
+ if (0 == (compression_methods & (1u<<method)))
+ continue; // client doesn't like this one, or we don't have it.
+ if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256,
+ diff_from, DIGEST256_LEN,
+ method) == CONSDIFF_AVAILABLE) {
+ tor_assert_nonfatal(result);
+ *compression_used_out = method;
+ return result;
+ }
+ }
+ } SMARTLIST_FOREACH_END(diff_from);
+
+ SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
+ if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from,
+ DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
+ tor_assert_nonfatal(result);
+ *compression_used_out = FALLBACK_COMPRESS_METHOD;
+ return result;
+ }
+ } SMARTLIST_FOREACH_END(diff_from);
+
+ return NULL;
+}
+
+/** Lookup the cached consensus document by the flavor found in <b>flav</b>.
+ * The preferred set of compression methods should be listed in the
+ * <b>compression_methods</b> bitfield. The compression method chosen (if any)
+ * is stored in <b>compression_used_out</b>. */
+static struct consensus_cache_entry_t *
+find_best_consensus(int flav,
+ unsigned compression_methods,
+ compress_method_t *compression_used_out)
+{
+ struct consensus_cache_entry_t *result = NULL;
+ unsigned u;
+
+ for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
+ compress_method_t method = srv_meth_pref_precompressed[u];
+
+ if (0 == (compression_methods & (1u<<method)))
+ continue;
+
+ if (consdiffmgr_find_consensus(&result, flav,
+ method) == CONSDIFF_AVAILABLE) {
+ tor_assert_nonfatal(result);
+ *compression_used_out = method;
+ return result;
+ }
+ }
+
+ if (consdiffmgr_find_consensus(&result, flav,
+ FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
+ tor_assert_nonfatal(result);
+ *compression_used_out = FALLBACK_COMPRESS_METHOD;
+ return result;
+ }
+
+ return NULL;
+}
+
+/** Try to find the best supported compression method possible from a given
+ * <b>compression_methods</b>. Return NO_METHOD if no mutually supported
+ * compression method could be found. */
+static compress_method_t
+find_best_compression_method(unsigned compression_methods, int stream)
+{
+ unsigned u;
+ compress_method_t *methods;
+ size_t length;
+
+ if (stream) {
+ methods = srv_meth_pref_streaming_compression;
+ length = ARRAY_LENGTH(srv_meth_pref_streaming_compression);
+ } else {
+ methods = srv_meth_pref_precompressed;
+ length = ARRAY_LENGTH(srv_meth_pref_precompressed);
+ }
+
+ for (u = 0; u < length; ++u) {
+ compress_method_t method = methods[u];
+ if (compression_methods & (1u<<method))
+ return method;
+ }
+
+ return NO_METHOD;
+}
+
+/** Check if any of the digests in <b>digests</b> matches the latest consensus
+ * flavor (given in <b>flavor</b>) that we have available. */
+static int
+digest_list_contains_best_consensus(consensus_flavor_t flavor,
+ const smartlist_t *digests)
+{
+ const networkstatus_t *ns = NULL;
+
+ if (digests == NULL)
+ return 0;
+
+ ns = networkstatus_get_latest_consensus_by_flavor(flavor);
+
+ if (ns == NULL)
+ return 0;
+
+ SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) {
+ if (tor_memeq(ns->digest_sha3_as_signed, digest, DIGEST256_LEN))
+ return 1;
+ } SMARTLIST_FOREACH_END(digest);
+
+ return 0;
+}
+
+/** Encodes the results of parsing a consensus request to figure out what
+ * consensus, and possibly what diffs, the user asked for. */
+typedef struct {
+ /** name of the flavor to retrieve. */
+ char *flavor;
+ /** flavor to retrive, as enum. */
+ consensus_flavor_t flav;
+ /** plus-separated list of authority fingerprints; see
+ * client_likes_consensus(). Aliases the URL in the request passed to
+ * parse_consensus_request(). */
+ const char *want_fps;
+ /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses
+ * to return a diff from. */
+ smartlist_t *diff_from_digests;
+ /** If true, never send a full consensus. If there is no diff, send
+ * a 404 instead. */
+ int diff_only;
+} parsed_consensus_request_t;
+
+/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since
+ * it is stack-allocated. */
+static void
+parsed_consensus_request_clear(parsed_consensus_request_t *req)
+{
+ if (!req)
+ return;
+ tor_free(req->flavor);
+ if (req->diff_from_digests) {
+ SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d));
+ smartlist_free(req->diff_from_digests);
+ }
+ memset(req, 0, sizeof(parsed_consensus_request_t));
+}
+
+/**
+ * Parse the URL and relevant headers of <b>args</b> for a current-consensus
+ * request to learn what flavor of consensus we want, what keys it must be
+ * signed with, and what diffs we would accept (or demand) instead. Return 0
+ * on success and -1 on failure.
+ */
+static int
+parse_consensus_request(parsed_consensus_request_t *out,
+ const get_handler_args_t *args)
+{
+ const char *url = args->url;
+ memset(out, 0, sizeof(parsed_consensus_request_t));
+ out->flav = FLAV_NS;
+
+ const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/";
+ const char CONSENSUS_FLAVORED_PREFIX[] =
+ "/tor/status-vote/current/consensus-";
+
+ /* figure out the flavor if any, and who we wanted to sign the thing */
+ const char *after_flavor = NULL;
+
+ if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
+ const char *f, *cp;
+ f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
+ cp = strchr(f, '/');
+ if (cp) {
+ after_flavor = cp+1;
+ out->flavor = tor_strndup(f, cp-f);
+ } else {
+ out->flavor = tor_strdup(f);
+ }
+ int flav = networkstatus_parse_flavor_name(out->flavor);
+ if (flav < 0)
+ flav = FLAV_NS;
+ out->flav = flav;
+ } else {
+ if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
+ after_flavor = url+strlen(CONSENSUS_URL_PREFIX);
+ }
+
+ /* see whether we've been asked explicitly for a diff from an older
+ * consensus. (The user might also have said that a diff would be okay,
+ * via X-Or-Diff-From-Consensus */
+ const char DIFF_COMPONENT[] = "diff/";
+ char *diff_hash_in_url = NULL;
+ if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) {
+ after_flavor += strlen(DIFF_COMPONENT);
+ const char *cp = strchr(after_flavor, '/');
+ if (cp) {
+ diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor);
+ out->want_fps = cp+1;
+ } else {
+ diff_hash_in_url = tor_strdup(after_flavor);
+ out->want_fps = NULL;
+ }
+ } else {
+ out->want_fps = after_flavor;
+ }
+
+ if (diff_hash_in_url) {
+ uint8_t diff_from[DIGEST256_LEN];
+ out->diff_from_digests = smartlist_new();
+ out->diff_only = 1;
+ int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL",
+ "rejecting");
+ tor_free(diff_hash_in_url);
+ if (ok) {
+ smartlist_add(out->diff_from_digests,
+ tor_memdup(diff_from, DIGEST256_LEN));
+ } else {
+ return -1;
+ }
+ } else {
+ parse_or_diff_from_header(&out->diff_from_digests, args->headers);
+ }
+
+ return 0;
+}
+
+/** Helper function for GET /tor/status-vote/current/consensus
+ */
+static int
+handle_get_current_consensus(dir_connection_t *conn,
+ const get_handler_args_t *args)
+{
+ const compress_method_t compress_method =
+ find_best_compression_method(args->compression_supported, 0);
+ const time_t if_modified_since = args->if_modified_since;
+ int clear_spool = 0;
+
+ /* v3 network status fetch. */
+ long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
+
+ time_t now = time(NULL);
+ parsed_consensus_request_t req;
+
+ if (parse_consensus_request(&req, args) < 0) {
+ write_short_http_response(conn, 404, "Couldn't parse request");
+ goto done;
+ }
+
+ if (digest_list_contains_best_consensus(req.flav,
+ req.diff_from_digests)) {
+ write_short_http_response(conn, 304, "Not modified");
+ geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
+ goto done;
+ }
+
+ struct consensus_cache_entry_t *cached_consensus = NULL;
+
+ compress_method_t compression_used = NO_METHOD;
+ if (req.diff_from_digests) {
+ cached_consensus = find_best_diff(req.diff_from_digests, req.flav,
+ args->compression_supported,
+ &compression_used);
+ }
+
+ if (req.diff_only && !cached_consensus) {
+ write_short_http_response(conn, 404, "No such diff available");
+ // XXXX warn_consensus_is_too_old(v, req.flavor, now);
+ geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+ goto done;
+ }
+
+ if (! cached_consensus) {
+ cached_consensus = find_best_consensus(req.flav,
+ args->compression_supported,
+ &compression_used);
+ }
+
+ time_t fresh_until, valid_until;
+ int have_fresh_until = 0, have_valid_until = 0;
+ if (cached_consensus) {
+ have_fresh_until =
+ !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
+ have_valid_until =
+ !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
+ }
+
+ if (cached_consensus && have_valid_until &&
+ !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
+ write_short_http_response(conn, 404, "Consensus is too old");
+ warn_consensus_is_too_old(cached_consensus, req.flavor, now);
+ geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+ goto done;
+ }
+
+ if (cached_consensus && req.want_fps &&
+ !client_likes_consensus(cached_consensus, req.want_fps)) {
+ write_short_http_response(conn, 404, "Consensus not signed by sufficient "
+ "number of requested authorities");
+ geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS);
+ goto done;
+ }
+
+ conn->spool = smartlist_new();
+ clear_spool = 1;
+ {
+ spooled_resource_t *spooled;
+ if (cached_consensus) {
+ spooled = spooled_resource_new_from_cache_entry(cached_consensus);
+ smartlist_add(conn->spool, spooled);
+ }
+ }
+
+ lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0;
+
+ size_t size_guess = 0;
+ int n_expired = 0;
+ dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since,
+ compress_method != NO_METHOD,
+ &size_guess,
+ &n_expired);
+
+ if (!smartlist_len(conn->spool) && !n_expired) {
+ write_short_http_response(conn, 404, "Not found");
+ geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+ goto done;
+ } else if (!smartlist_len(conn->spool)) {
+ write_short_http_response(conn, 304, "Not modified");
+ geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
+ goto done;
+ }
+
+ if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+ log_debug(LD_DIRSERV,
+ "Client asked for network status lists, but we've been "
+ "writing too many bytes lately. Sending 503 Dir busy.");
+ write_short_http_response(conn, 503, "Directory busy, try again later");
+ geoip_note_ns_response(GEOIP_REJECT_BUSY);
+ goto done;
+ }
+
+ tor_addr_t addr;
+ if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
+ geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS,
+ &addr, NULL,
+ time(NULL));
+ geoip_note_ns_response(GEOIP_SUCCESS);
+ /* Note that a request for a network status has started, so that we
+ * can measure the download time later on. */
+ if (conn->dirreq_id)
+ geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED);
+ else
+ geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess,
+ DIRREQ_DIRECT);
+ }
+
+ /* Use this header to tell caches that the response depends on the
+ * X-Or-Diff-From-Consensus header (or lack thereof). */
+ const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n";
+
+ clear_spool = 0;
+
+ // The compress_method might have been NO_METHOD, but we store the data
+ // compressed. Decompress them using `compression_used`. See fallback code in
+ // find_best_consensus() and find_best_diff().
+ write_http_response_headers(conn, -1,
+ compress_method == NO_METHOD ?
+ NO_METHOD : compression_used,
+ vary_header,
+ smartlist_len(conn->spool) == 1 ? lifetime : 0);
+
+ if (compress_method == NO_METHOD && smartlist_len(conn->spool))
+ conn->compress_state = tor_compress_new(0, compression_used,
+ HIGH_COMPRESSION);
+
+ /* Prime the connection with some data. */
+ const int initial_flush_result = connection_dirserv_flushed_some(conn);
+ tor_assert_nonfatal(initial_flush_result == 0);
+ goto done;
+
+ done:
+ parsed_consensus_request_clear(&req);
+ if (clear_spool) {
+ dir_conn_clear_spool(conn);
+ }
+ return 0;
+}
+
+/** Helper function for GET /tor/status-vote/{current,next}/...
+ */
+static int
+handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
+{
+ const char *url = args->url;
+ {
+ ssize_t body_len = 0;
+ ssize_t estimated_len = 0;
+ int lifetime = 60; /* XXXX?? should actually use vote intervals. */
+ /* This smartlist holds strings that we can compress on the fly. */
+ smartlist_t *items = smartlist_new();
+ /* This smartlist holds cached_dir_t objects that have a precompressed
+ * deflated version. */
+ smartlist_t *dir_items = smartlist_new();
+ dirvote_dirreq_get_status_vote(url, items, dir_items);
+ if (!smartlist_len(dir_items) && !smartlist_len(items)) {
+ write_short_http_response(conn, 404, "Not found");
+ goto vote_done;
+ }
+
+ /* We're sending items from at most one kind of source */
+ tor_assert_nonfatal(smartlist_len(items) == 0 ||
+ smartlist_len(dir_items) == 0);
+
+ int streaming;
+ unsigned mask;
+ if (smartlist_len(items)) {
+ /* We're taking strings and compressing them on the fly. */
+ streaming = 1;
+ mask = ~0u;
+ } else {
+ /* We're taking cached_dir_t objects. We only have them uncompressed
+ * or deflated. */
+ streaming = 0;
+ mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD);
+ }
+ const compress_method_t compress_method = find_best_compression_method(
+ args->compression_supported&mask, streaming);
+
+ SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
+ body_len += compress_method != NO_METHOD ?
+ d->dir_compressed_len : d->dir_len);
+ estimated_len += body_len;
+ SMARTLIST_FOREACH(items, const char *, item, {
+ size_t ln = strlen(item);
+ if (compress_method != NO_METHOD) {
+ estimated_len += ln/2;
+ } else {
+ body_len += ln; estimated_len += ln;
+ }
+ });
+
+ if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
+ write_short_http_response(conn, 503, "Directory busy, try again later");
+ goto vote_done;
+ }
+ write_http_response_header(conn, body_len ? body_len : -1,
+ compress_method,
+ lifetime);
+
+ if (smartlist_len(items)) {
+ if (compress_method != NO_METHOD) {
+ conn->compress_state = tor_compress_new(1, compress_method,
+ choose_compression_level(estimated_len));
+ SMARTLIST_FOREACH(items, const char *, c,
+ connection_buf_add_compress(c, strlen(c), conn, 0));
+ connection_buf_add_compress("", 0, conn, 1);
+ } else {
+ SMARTLIST_FOREACH(items, const char *, c,
+ connection_buf_add(c, strlen(c), TO_CONN(conn)));
+ }
+ } else {
+ SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
+ connection_buf_add(compress_method != NO_METHOD ?
+ d->dir_compressed : d->dir,
+ compress_method != NO_METHOD ?
+ d->dir_compressed_len : d->dir_len,
+ TO_CONN(conn)));
+ }
+ vote_done:
+ smartlist_free(items);
+ smartlist_free(dir_items);
+ goto done;
+ }
+ done:
+ return 0;
+}
+
+/** Helper function for GET /tor/micro/d/...
+ */
+static int
+handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
+{
+ const char *url = args->url;
+ const compress_method_t compress_method =
+ find_best_compression_method(args->compression_supported, 1);
+ int clear_spool = 1;
+ {
+ conn->spool = smartlist_new();
+
+ dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"),
+ DIR_SPOOL_MICRODESC,
+ conn->spool, NULL,
+ DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
+
+ size_t size_guess = 0;
+ dirserv_spool_remove_missing_and_guess_size(conn, 0,
+ compress_method != NO_METHOD,
+ &size_guess, NULL);
+ if (smartlist_len(conn->spool) == 0) {
+ write_short_http_response(conn, 404, "Not found");
+ goto done;
+ }
+ if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+ log_info(LD_DIRSERV,
+ "Client asked for server descriptors, but we've been "
+ "writing too many bytes lately. Sending 503 Dir busy.");
+ write_short_http_response(conn, 503, "Directory busy, try again later");
+ goto done;
+ }
+
+ clear_spool = 0;
+ write_http_response_header(conn, -1,
+ compress_method,
+ MICRODESC_CACHE_LIFETIME);
+
+ if (compress_method != NO_METHOD)
+ conn->compress_state = tor_compress_new(1, compress_method,
+ choose_compression_level(size_guess));
+
+ const int initial_flush_result = connection_dirserv_flushed_some(conn);
+ tor_assert_nonfatal(initial_flush_result == 0);
+ goto done;
+ }
+
+ done:
+ if (clear_spool) {
+ dir_conn_clear_spool(conn);
+ }
+ return 0;
+}
+
+/** Helper function for GET /tor/{server,extra}/...
+ */
+static int
+handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
+{
+ const char *url = args->url;
+ const compress_method_t compress_method =
+ find_best_compression_method(args->compression_supported, 1);
+ const or_options_t *options = get_options();
+ int clear_spool = 1;
+ if (!strcmpstart(url,"/tor/server/") ||
+ (!options->BridgeAuthoritativeDir &&
+ !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
+ int res;
+ const char *msg = NULL;
+ int cache_lifetime = 0;
+ int is_extra = !strcmpstart(url,"/tor/extra/");
+ url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/");
+ dir_spool_source_t source;
+ time_t publish_cutoff = 0;
+ if (!strcmpstart(url, "d/")) {
+ source =
+ is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST;
+ } else {
+ source =
+ is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP;
+ /* We only want to apply a publish cutoff when we're requesting
+ * resources by fingerprint. */
+ publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
+ }
+
+ conn->spool = smartlist_new();
+ res = dirserv_get_routerdesc_spool(conn->spool, url,
+ source,
+ connection_dir_is_encrypted(conn),
+ &msg);
+
+ if (!strcmpstart(url, "all")) {
+ cache_lifetime = FULL_DIR_CACHE_LIFETIME;
+ } else if (smartlist_len(conn->spool) == 1) {
+ cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
+ }
+
+ size_t size_guess = 0;
+ int n_expired = 0;
+ dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff,
+ compress_method != NO_METHOD,
+ &size_guess, &n_expired);
+
+ /* If we are the bridge authority and the descriptor is a bridge
+ * descriptor, remember that we served this descriptor for desc stats. */
+ /* XXXX it's a bit of a kludge to have this here. */
+ if (get_options()->BridgeAuthoritativeDir &&
+ source == DIR_SPOOL_SERVER_BY_FP) {
+ SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) {
+ const routerinfo_t *router =
+ router_get_by_id_digest((const char *)spooled->digest);
+ /* router can be NULL here when the bridge auth is asked for its own
+ * descriptor. */
+ if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
+ rep_hist_note_desc_served(router->cache_info.identity_digest);
+ } SMARTLIST_FOREACH_END(spooled);
+ }
+
+ if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) {
+ if (msg == NULL)
+ msg = "Not found";
+ write_short_http_response(conn, 404, msg);
+ } else {
+ if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+ log_info(LD_DIRSERV,
+ "Client asked for server descriptors, but we've been "
+ "writing too many bytes lately. Sending 503 Dir busy.");
+ write_short_http_response(conn, 503,
+ "Directory busy, try again later");
+ dir_conn_clear_spool(conn);
+ goto done;
+ }
+ write_http_response_header(conn, -1, compress_method, cache_lifetime);
+ if (compress_method != NO_METHOD)
+ conn->compress_state = tor_compress_new(1, compress_method,
+ choose_compression_level(size_guess));
+ clear_spool = 0;
+ /* Prime the connection with some data. */
+ int initial_flush_result = connection_dirserv_flushed_some(conn);
+ tor_assert_nonfatal(initial_flush_result == 0);
+ }
+ goto done;
+ }
+ done:
+ if (clear_spool)
+ dir_conn_clear_spool(conn);
+ return 0;
+}
+
+/** Helper function for GET /tor/keys/...
+ */
+static int
+handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
+{
+ const char *url = args->url;
+ const compress_method_t compress_method =
+ find_best_compression_method(args->compression_supported, 1);
+ const time_t if_modified_since = args->if_modified_since;
+ {
+ smartlist_t *certs = smartlist_new();
+ ssize_t len = -1;
+ if (!strcmp(url, "/tor/keys/all")) {
+ authority_cert_get_all(certs);
+ } else if (!strcmp(url, "/tor/keys/authority")) {
+ authority_cert_t *cert = get_my_v3_authority_cert();
+ if (cert)
+ smartlist_add(certs, cert);
+ } else if (!strcmpstart(url, "/tor/keys/fp/")) {
+ smartlist_t *fps = smartlist_new();
+ dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"),
+ fps, NULL,
+ DSR_HEX|DSR_SORT_UNIQ);
+ SMARTLIST_FOREACH(fps, char *, d, {
+ authority_cert_t *c = authority_cert_get_newest_by_id(d);
+ if (c) smartlist_add(certs, c);
+ tor_free(d);
+ });
+ smartlist_free(fps);
+ } else if (!strcmpstart(url, "/tor/keys/sk/")) {
+ smartlist_t *fps = smartlist_new();
+ dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"),
+ fps, NULL,
+ DSR_HEX|DSR_SORT_UNIQ);
+ SMARTLIST_FOREACH(fps, char *, d, {
+ authority_cert_t *c = authority_cert_get_by_sk_digest(d);
+ if (c) smartlist_add(certs, c);
+ tor_free(d);
+ });
+ smartlist_free(fps);
+ } else if (!strcmpstart(url, "/tor/keys/fp-sk/")) {
+ smartlist_t *fp_sks = smartlist_new();
+ dir_split_resource_into_fingerprint_pairs(url+strlen("/tor/keys/fp-sk/"),
+ fp_sks);
+ SMARTLIST_FOREACH(fp_sks, fp_pair_t *, pair, {
+ authority_cert_t *c = authority_cert_get_by_digests(pair->first,
+ pair->second);
+ if (c) smartlist_add(certs, c);
+ tor_free(pair);
+ });
+ smartlist_free(fp_sks);
+ } else {
+ write_short_http_response(conn, 400, "Bad request");
+ goto keys_done;
+ }
+ if (!smartlist_len(certs)) {
+ write_short_http_response(conn, 404, "Not found");
+ goto keys_done;
+ }
+ SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+ if (c->cache_info.published_on < if_modified_since)
+ SMARTLIST_DEL_CURRENT(certs, c));
+ if (!smartlist_len(certs)) {
+ write_short_http_response(conn, 304, "Not modified");
+ goto keys_done;
+ }
+ len = 0;
+ SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+ len += c->cache_info.signed_descriptor_len);
+
+ if (global_write_bucket_low(TO_CONN(conn),
+ compress_method != NO_METHOD ? len/2 : len,
+ 2)) {
+ write_short_http_response(conn, 503, "Directory busy, try again later");
+ goto keys_done;
+ }
+
+ write_http_response_header(conn,
+ compress_method != NO_METHOD ? -1 : len,
+ compress_method,
+ 60*60);
+ if (compress_method != NO_METHOD) {
+ conn->compress_state = tor_compress_new(1, compress_method,
+ choose_compression_level(len));
+ SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+ connection_buf_add_compress(
+ c->cache_info.signed_descriptor_body,
+ c->cache_info.signed_descriptor_len,
+ conn, 0));
+ connection_buf_add_compress("", 0, conn, 1);
+ } else {
+ SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+ connection_buf_add(c->cache_info.signed_descriptor_body,
+ c->cache_info.signed_descriptor_len,
+ TO_CONN(conn)));
+ }
+ keys_done:
+ smartlist_free(certs);
+ goto done;
+ }
+ done:
+ return 0;
+}
+
+/** Helper function for GET /tor/rendezvous2/
+ */
+static int
+handle_get_hs_descriptor_v2(dir_connection_t *conn,
+ const get_handler_args_t *args)
+{
+ const char *url = args->url;
+ if (connection_dir_is_encrypted(conn)) {
+ /* Handle v2 rendezvous descriptor fetch request. */
+ const char *descp;
+ const char *query = url + strlen("/tor/rendezvous2/");
+ if (rend_valid_descriptor_id(query)) {
+ log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'",
+ safe_str(escaped(query)));
+ switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
+ case 1: /* valid */
+ write_http_response_header(conn, strlen(descp), NO_METHOD, 0);
+ connection_buf_add(descp, strlen(descp), TO_CONN(conn));
+ break;
+ case 0: /* well-formed but not present */
+ write_short_http_response(conn, 404, "Not found");
+ break;
+ case -1: /* not well-formed */
+ write_short_http_response(conn, 400, "Bad request");
+ break;
+ }
+ } else { /* not well-formed */
+ write_short_http_response(conn, 400, "Bad request");
+ }
+ goto done;
+ } else {
+ /* Not encrypted! */
+ write_short_http_response(conn, 404, "Not found");
+ }
+ done:
+ return 0;
+}
+
+/** Helper function for GET /tor/hs/3/<z>. Only for version 3.
+ */
+STATIC int
+handle_get_hs_descriptor_v3(dir_connection_t *conn,
+ const get_handler_args_t *args)
+{
+ int retval;
+ const char *desc_str = NULL;
+ const char *pubkey_str = NULL;
+ const char *url = args->url;
+
+ /* Reject unencrypted dir connections */
+ if (!connection_dir_is_encrypted(conn)) {
+ write_short_http_response(conn, 404, "Not found");
+ goto done;
+ }
+
+ /* After the path prefix follows the base64 encoded blinded pubkey which we
+ * use to get the descriptor from the cache. Skip the prefix and get the
+ * pubkey. */
+ tor_assert(!strcmpstart(url, "/tor/hs/3/"));
+ pubkey_str = url + strlen("/tor/hs/3/");
+ retval = hs_cache_lookup_as_dir(HS_VERSION_THREE,
+ pubkey_str, &desc_str);
+ if (retval <= 0 || desc_str == NULL) {
+ write_short_http_response(conn, 404, "Not found");
+ goto done;
+ }
+
+ /* Found requested descriptor! Pass it to this nice client. */
+ write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0);
+ connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn));
+
+ done:
+ return 0;
+}
+
+/** Helper function for GET /tor/networkstatus-bridges
+ */
+static int
+handle_get_networkstatus_bridges(dir_connection_t *conn,
+ const get_handler_args_t *args)
+{
+ const char *headers = args->headers;
+
+ const or_options_t *options = get_options();
+ if (options->BridgeAuthoritativeDir &&
+ options->BridgePassword_AuthDigest_ &&
+ connection_dir_is_encrypted(conn)) {
+ char *status;
+ char digest[DIGEST256_LEN];
+
+ char *header = http_get_header(headers, "Authorization: Basic ");
+ if (header)
+ crypto_digest256(digest, header, strlen(header), DIGEST_SHA256);
+
+ /* now make sure the password is there and right */
+ if (!header ||
+ tor_memneq(digest,
+ options->BridgePassword_AuthDigest_, DIGEST256_LEN)) {
+ write_short_http_response(conn, 404, "Not found");
+ tor_free(header);
+ goto done;
+ }
+ tor_free(header);
+
+ /* all happy now. send an answer. */
+ status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
+ size_t dlen = strlen(status);
+ write_http_response_header(conn, dlen, NO_METHOD, 0);
+ connection_buf_add(status, dlen, TO_CONN(conn));
+ tor_free(status);
+ goto done;
+ }
+ done:
+ return 0;
+}
+
+/** Helper function for GET robots.txt or /tor/robots.txt */
+static int
+handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
+{
+ (void)args;
+ {
+ const char robots[] = "User-agent: *\r\nDisallow: /\r\n";
+ size_t len = strlen(robots);
+ write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME);
+ connection_buf_add(robots, len, TO_CONN(conn));
+ }
+ return 0;
+}
+
+/* Given the <b>url</b> from a POST request, try to extract the version number
+ * using the provided <b>prefix</b>. The version should be after the prefix and
+ * ending with the separator "/". For instance:
+ * /tor/hs/3/publish
+ *
+ * On success, <b>end_pos</b> points to the position right after the version
+ * was found. On error, it is set to NULL.
+ *
+ * Return version on success else negative value. */
+STATIC int
+parse_hs_version_from_post(const char *url, const char *prefix,
+ const char **end_pos)
+{
+ int ok;
+ unsigned long version;
+ const char *start;
+ char *end = NULL;
+
+ tor_assert(url);
+ tor_assert(prefix);
+ tor_assert(end_pos);
+
+ /* Check if the prefix does start the url. */
+ if (strcmpstart(url, prefix)) {
+ goto err;
+ }
+ /* Move pointer to the end of the prefix string. */
+ start = url + strlen(prefix);
+ /* Try this to be the HS version and if we are still at the separator, next
+ * will be move to the right value. */
+ version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end);
+ if (!ok) {
+ goto err;
+ }
+
+ *end_pos = end;
+ return (int) version;
+ err:
+ *end_pos = NULL;
+ return -1;
+}
+
+/* Handle the POST request for a hidden service descripror. The request is in
+ * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success
+ * else return 400 indicating a bad request. */
+STATIC int
+handle_post_hs_descriptor(const char *url, const char *body)
+{
+ int version;
+ const char *end_pos;
+
+ tor_assert(url);
+ tor_assert(body);
+
+ version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos);
+ if (version < 0) {
+ goto err;
+ }
+
+ /* We have a valid version number, now make sure it's a publish request. Use
+ * the end position just after the version and check for the command. */
+ if (strcmpstart(end_pos, "/publish")) {
+ goto err;
+ }
+
+ switch (version) {
+ case HS_VERSION_THREE:
+ if (hs_cache_store_as_dir(body) < 0) {
+ goto err;
+ }
+ log_info(LD_REND, "Publish request for HS descriptor handled "
+ "successfully.");
+ break;
+ default:
+ /* Unsupported version, return a bad request. */
+ goto err;
+ }
+
+ return 200;
+ err:
+ /* Bad request. */
+ return 400;
+}
+
+/** Helper function: called when a dirserver gets a complete HTTP POST
+ * request. Look for an uploaded server descriptor or rendezvous
+ * service descriptor. On finding one, process it and write a
+ * response into conn-\>outbuf. If the request is unrecognized, send a
+ * 400. Always return 0. */
+MOCK_IMPL(STATIC int,
+directory_handle_command_post,(dir_connection_t *conn, const char *headers,
+ const char *body, size_t body_len))
+{
+ char *url = NULL;
+ const or_options_t *options = get_options();
+
+ log_debug(LD_DIRSERV,"Received POST command.");
+
+ conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
+
+ if (!public_server_mode(options)) {
+ log_info(LD_DIR, "Rejected dir post request from %s "
+ "since we're not a public relay.", conn->base_.address);
+ write_short_http_response(conn, 503, "Not acting as a public relay");
+ goto done;
+ }
+
+ if (parse_http_url(headers, &url) < 0) {
+ write_short_http_response(conn, 400, "Bad request");
+ return 0;
+ }
+ log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
+
+ /* Handle v2 rendezvous service publish request. */
+ if (connection_dir_is_encrypted(conn) &&
+ !strcmpstart(url,"/tor/rendezvous2/publish")) {
+ if (rend_cache_store_v2_desc_as_dir(body) < 0) {
+ log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.",
+ (int)body_len, conn->base_.address);
+ write_short_http_response(conn, 400,
+ "Invalid v2 service descriptor rejected");
+ } else {
+ write_short_http_response(conn, 200, "Service descriptor (v2) stored");
+ log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted");
+ }
+ goto done;
+ }
+
+ /* Handle HS descriptor publish request. */
+ /* XXX: This should be disabled with a consensus param until we want to
+ * the prop224 be deployed and thus use. */
+ if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
+ const char *msg = "HS descriptor stored successfully.";
+
+ /* We most probably have a publish request for an HS descriptor. */
+ int code = handle_post_hs_descriptor(url, body);
+ if (code != 200) {
+ msg = "Invalid HS descriptor. Rejected.";
+ }
+ write_short_http_response(conn, code, msg);
+ goto done;
+ }
+
+ if (!authdir_mode(options)) {
+ /* we just provide cached directories; we don't want to
+ * receive anything. */
+ write_short_http_response(conn, 400, "Nonauthoritative directory does not "
+ "accept posted server descriptors");
+ goto done;
+ }
+
+ if (authdir_mode(options) &&
+ !strcmp(url,"/tor/")) { /* server descriptor post */
+ const char *msg = "[None]";
+ uint8_t purpose = authdir_mode_bridge(options) ?
+ ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
+ was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose,
+ conn->base_.address, &msg);
+ tor_assert(msg);
+
+ if (r == ROUTER_ADDED_SUCCESSFULLY) {
+ write_short_http_response(conn, 200, msg);
+ } else if (WRA_WAS_OUTDATED(r)) {
+ write_http_response_header_impl(conn, -1, NULL, NULL,
+ "X-Descriptor-Not-New: Yes\r\n", -1);
+ } else {
+ log_info(LD_DIRSERV,
+ "Rejected router descriptor or extra-info from %s "
+ "(\"%s\").",
+ conn->base_.address, msg);
+ write_short_http_response(conn, 400, msg);
+ }
+ goto done;
+ }
+
+ if (authdir_mode_v3(options) &&
+ !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */
+ const char *msg = "OK";
+ int status;
+ if (dirvote_add_vote(body, &msg, &status)) {
+ write_short_http_response(conn, status, "Vote stored");
+ } else {
+ tor_assert(msg);
+ log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").",
+ conn->base_.address, msg);
+ write_short_http_response(conn, status, msg);
+ }
+ goto done;
+ }
+
+ if (authdir_mode_v3(options) &&
+ !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */
+ const char *msg = NULL;
+ if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) {
+ write_short_http_response(conn, 200, msg?msg:"Signatures stored");
+ } else {
+ log_warn(LD_DIR, "Unable to store signatures posted by %s: %s",
+ conn->base_.address, msg?msg:"???");
+ write_short_http_response(conn, 400,
+ msg?msg:"Unable to store signatures");
+ }
+ goto done;
+ }
+
+ /* we didn't recognize the url */
+ write_short_http_response(conn, 404, "Not found");
+
+ done:
+ tor_free(url);
+ return 0;
+}
+
+/** If <b>headers</b> indicates that a proxy was involved, then rewrite
+ * <b>conn</b>-\>address to describe our best guess of the address that
+ * originated this HTTP request. */
+static void
+http_set_address_origin(const char *headers, connection_t *conn)
+{
+ char *fwd;
+
+ fwd = http_get_header(headers, "Forwarded-For: ");
+ if (!fwd)
+ fwd = http_get_header(headers, "X-Forwarded-For: ");
+ if (fwd) {
+ tor_addr_t toraddr;
+ if (tor_addr_parse(&toraddr,fwd) == -1 ||
+ tor_addr_is_internal(&toraddr,0)) {
+ log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd));
+ tor_free(fwd);
+ return;
+ }
+
+ tor_free(conn->address);
+ conn->address = tor_strdup(fwd);
+ tor_free(fwd);
+ }
+}
+
+/** Called when a dirserver receives data on a directory connection;
+ * looks for an HTTP request. If the request is complete, remove it
+ * from the inbuf, try to process it; otherwise, leave it on the
+ * buffer. Return a 0 on success, or -1 on error.
+ */
+int
+directory_handle_command(dir_connection_t *conn)
+{
+ char *headers=NULL, *body=NULL;
+ size_t body_len=0;
+ int r;
+
+ tor_assert(conn);
+ tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+ switch (connection_fetch_from_buf_http(TO_CONN(conn),
+ &headers, MAX_HEADERS_SIZE,
+ &body, &body_len, MAX_DIR_UL_SIZE, 0)) {
+ case -1: /* overflow */
+ log_warn(LD_DIRSERV,
+ "Request too large from address '%s' to DirPort. Closing.",
+ safe_str(conn->base_.address));
+ return -1;
+ case 0:
+ log_debug(LD_DIRSERV,"command not all here yet.");
+ return 0;
+ /* case 1, fall through */
+ }
+
+ http_set_address_origin(headers, TO_CONN(conn));
+ // we should escape headers here as well,
+ // but we can't call escaped() twice, as it uses the same buffer
+ //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body));
+
+ if (!strncasecmp(headers,"GET",3))
+ r = directory_handle_command_get(conn, headers, body, body_len);
+ else if (!strncasecmp(headers,"POST",4))
+ r = directory_handle_command_post(conn, headers, body, body_len);
+ else {
+ log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+ "Got headers %s with unknown command. Closing.",
+ escaped(headers));
+ r = -1;
+ }
+
+ tor_free(headers); tor_free(body);
+ return r;
+}
diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h
new file mode 100644
index 000000000..f05780375
--- /dev/null
+++ b/src/feature/dircache/dircache.h
@@ -0,0 +1,43 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dircache.h
+ * \brief Header file for dircache.c.
+ **/
+
+#ifndef TOR_DIRCACHE_H
+#define TOR_DIRCACHE_H
+
+int directory_handle_command(dir_connection_t *conn);
+
+#ifdef DIRCACHE_PRIVATE
+MOCK_DECL(STATIC int, directory_handle_command_get,(dir_connection_t *conn,
+ const char *headers,
+ const char *req_body,
+ size_t req_body_len));
+MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn,
+ const char *headers,
+ const char *body,
+ size_t body_len));
+
+STATIC int handle_post_hs_descriptor(const char *url, const char *body);
+enum compression_level_t;
+STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes);
+
+struct get_handler_args_t;
+STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn,
+ const struct get_handler_args_t *args);
+
+STATIC int parse_http_url(const char *headers, char **url);
+
+STATIC int parse_hs_version_from_post(const char *url, const char *prefix,
+ const char **end_pos);
+
+STATIC unsigned parse_accept_encoding_header(const char *h);
+#endif
+
+#endif /* !defined(TOR_DIRCACHE_H) */
diff --git a/src/feature/dircache/directory.h b/src/feature/dircache/directory.h
deleted file mode 100644
index 54ea0c584..000000000
--- a/src/feature/dircache/directory.h
+++ /dev/null
@@ -1,347 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2018, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file directory.h
- * \brief Header file for directory.c.
- **/
-
-#ifndef TOR_DIRECTORY_H
-#define TOR_DIRECTORY_H
-
-#include "feature/hs/hs_ident.h"
-enum compress_method_t;
-
-dir_connection_t *TO_DIR_CONN(connection_t *c);
-
-#define DIR_CONN_STATE_MIN_ 1
-/** State for connection to directory server: waiting for connect(). */
-#define DIR_CONN_STATE_CONNECTING 1
-/** State for connection to directory server: sending HTTP request. */
-#define DIR_CONN_STATE_CLIENT_SENDING 2
-/** State for connection to directory server: reading HTTP response. */
-#define DIR_CONN_STATE_CLIENT_READING 3
-/** State for connection to directory server: happy and finished. */
-#define DIR_CONN_STATE_CLIENT_FINISHED 4
-/** State for connection at directory server: waiting for HTTP request. */
-#define DIR_CONN_STATE_SERVER_COMMAND_WAIT 5
-/** State for connection at directory server: sending HTTP response. */
-#define DIR_CONN_STATE_SERVER_WRITING 6
-#define DIR_CONN_STATE_MAX_ 6
-
-#define DIR_PURPOSE_MIN_ 4
-/** A connection to a directory server: set after a v2 rendezvous
- * descriptor is downloaded. */
-#define DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 4
-/** A connection to a directory server: download one or more server
- * descriptors. */
-#define DIR_PURPOSE_FETCH_SERVERDESC 6
-/** A connection to a directory server: download one or more extra-info
- * documents. */
-#define DIR_PURPOSE_FETCH_EXTRAINFO 7
-/** A connection to a directory server: upload a server descriptor. */
-#define DIR_PURPOSE_UPLOAD_DIR 8
-/** A connection to a directory server: upload a v3 networkstatus vote. */
-#define DIR_PURPOSE_UPLOAD_VOTE 10
-/** A connection to a directory server: upload a v3 consensus signature */
-#define DIR_PURPOSE_UPLOAD_SIGNATURES 11
-/** A connection to a directory server: download one or more v3 networkstatus
- * votes. */
-#define DIR_PURPOSE_FETCH_STATUS_VOTE 12
-/** A connection to a directory server: download a v3 detached signatures
- * object for a consensus. */
-#define DIR_PURPOSE_FETCH_DETACHED_SIGNATURES 13
-/** A connection to a directory server: download a v3 networkstatus
- * consensus. */
-#define DIR_PURPOSE_FETCH_CONSENSUS 14
-/** A connection to a directory server: download one or more directory
- * authority certificates. */
-#define DIR_PURPOSE_FETCH_CERTIFICATE 15
-
-/** Purpose for connection at a directory server. */
-#define DIR_PURPOSE_SERVER 16
-/** A connection to a hidden service directory server: upload a v2 rendezvous
- * descriptor. */
-#define DIR_PURPOSE_UPLOAD_RENDDESC_V2 17
-/** A connection to a hidden service directory server: download a v2 rendezvous
- * descriptor. */
-#define DIR_PURPOSE_FETCH_RENDDESC_V2 18
-/** A connection to a directory server: download a microdescriptor. */
-#define DIR_PURPOSE_FETCH_MICRODESC 19
-/** A connection to a hidden service directory: upload a v3 descriptor. */
-#define DIR_PURPOSE_UPLOAD_HSDESC 20
-/** A connection to a hidden service directory: fetch a v3 descriptor. */
-#define DIR_PURPOSE_FETCH_HSDESC 21
-/** A connection to a directory server: set after a hidden service descriptor
- * is downloaded. */
-#define DIR_PURPOSE_HAS_FETCHED_HSDESC 22
-#define DIR_PURPOSE_MAX_ 22
-
-/** True iff <b>p</b> is a purpose corresponding to uploading
- * data to a directory server. */
-#define DIR_PURPOSE_IS_UPLOAD(p) \
- ((p)==DIR_PURPOSE_UPLOAD_DIR || \
- (p)==DIR_PURPOSE_UPLOAD_VOTE || \
- (p)==DIR_PURPOSE_UPLOAD_SIGNATURES || \
- (p)==DIR_PURPOSE_UPLOAD_RENDDESC_V2 || \
- (p)==DIR_PURPOSE_UPLOAD_HSDESC)
-
-int directories_have_accepted_server_descriptor(void);
-void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
- dirinfo_type_t type, const char *payload,
- size_t payload_len, size_t extrainfo_len);
-MOCK_DECL(void, directory_get_from_dirserver, (
- uint8_t dir_purpose,
- uint8_t router_purpose,
- const char *resource,
- int pds_flags,
- download_want_authority_t want_authority));
-void directory_get_from_all_authorities(uint8_t dir_purpose,
- uint8_t router_purpose,
- const char *resource);
-
-/** Enumeration of ways to connect to a directory server */
-typedef enum {
- /** Default: connect over a one-hop Tor circuit. Relays fall back to direct
- * DirPort connections, clients, onion services, and bridges do not */
- DIRIND_ONEHOP=0,
- /** Connect over a multi-hop anonymizing Tor circuit */
- DIRIND_ANONYMOUS=1,
- /** Connect to the DirPort directly */
- DIRIND_DIRECT_CONN,
- /** Connect over a multi-hop anonymizing Tor circuit to our dirport */
- DIRIND_ANON_DIRPORT,
-} dir_indirection_t;
-
-int directory_must_use_begindir(const or_options_t *options);
-
-/**
- * A directory_request_t describes the information about a directory request
- * at the client side. It describes what we're going to ask for, which
- * directory we're going to ask for it, how we're going to contact that
- * directory, and (in some cases) what to do with it when we're done.
- */
-typedef struct directory_request_t directory_request_t;
-directory_request_t *directory_request_new(uint8_t dir_purpose);
-void directory_request_free_(directory_request_t *req);
-#define directory_request_free(req) \
- FREE_AND_NULL(directory_request_t, directory_request_free_, (req))
-void directory_request_set_or_addr_port(directory_request_t *req,
- const tor_addr_port_t *p);
-void directory_request_set_dir_addr_port(directory_request_t *req,
- const tor_addr_port_t *p);
-void directory_request_set_directory_id_digest(directory_request_t *req,
- const char *digest);
-struct circuit_guard_state_t;
-void directory_request_set_guard_state(directory_request_t *req,
- struct circuit_guard_state_t *state);
-void directory_request_set_router_purpose(directory_request_t *req,
- uint8_t router_purpose);
-void directory_request_set_indirection(directory_request_t *req,
- dir_indirection_t indirection);
-void directory_request_set_resource(directory_request_t *req,
- const char *resource);
-void directory_request_set_payload(directory_request_t *req,
- const char *payload,
- size_t payload_len);
-void directory_request_set_if_modified_since(directory_request_t *req,
- time_t if_modified_since);
-void directory_request_set_rend_query(directory_request_t *req,
- const rend_data_t *query);
-void directory_request_upload_set_hs_ident(directory_request_t *req,
- const hs_ident_dir_conn_t *ident);
-void directory_request_fetch_set_hs_ident(directory_request_t *req,
- const hs_ident_dir_conn_t *ident);
-
-void directory_request_set_routerstatus(directory_request_t *req,
- const routerstatus_t *rs);
-void directory_request_add_header(directory_request_t *req,
- const char *key,
- const char *val);
-MOCK_DECL(void, directory_initiate_request, (directory_request_t *request));
-
-int parse_http_response(const char *headers, int *code, time_t *date,
- enum compress_method_t *compression, char **response);
-int parse_http_command(const char *headers,
- char **command_out, char **url_out);
-char *http_get_header(const char *headers, const char *which);
-
-int connection_dir_is_encrypted(const dir_connection_t *conn);
-int connection_dir_reached_eof(dir_connection_t *conn);
-int connection_dir_process_inbuf(dir_connection_t *conn);
-int connection_dir_finished_flushing(dir_connection_t *conn);
-int connection_dir_finished_connecting(dir_connection_t *conn);
-void connection_dir_about_to_close(dir_connection_t *dir_conn);
-
-#define DSR_HEX (1<<0)
-#define DSR_BASE64 (1<<1)
-#define DSR_DIGEST256 (1<<2)
-#define DSR_SORT_UNIQ (1<<3)
-int dir_split_resource_into_fingerprints(const char *resource,
- smartlist_t *fp_out, int *compressed_out,
- int flags);
-enum dir_spool_source_t;
-int dir_split_resource_into_spoolable(const char *resource,
- enum dir_spool_source_t source,
- smartlist_t *spool_out,
- int *compressed_out,
- int flags);
-int dir_split_resource_into_fingerprint_pairs(const char *res,
- smartlist_t *pairs_out);
-char *directory_dump_request_log(void);
-void note_request(const char *key, size_t bytes);
-int router_supports_extrainfo(const char *identity_digest, int is_authority);
-
-time_t download_status_increment_failure(download_status_t *dls,
- int status_code, const char *item,
- int server, time_t now);
-time_t download_status_increment_attempt(download_status_t *dls,
- const char *item, time_t now);
-/** Increment the failure count of the download_status_t <b>dls</b>, with
- * the optional status code <b>sc</b>. */
-#define download_status_failed(dls, sc) \
- download_status_increment_failure((dls), (sc), NULL, \
- dir_server_mode(get_options()), \
- time(NULL))
-
-void download_status_reset(download_status_t *dls);
-int download_status_is_ready(download_status_t *dls, time_t now);
-time_t download_status_get_next_attempt_at(const download_status_t *dls);
-void download_status_mark_impossible(download_status_t *dl);
-
-int download_status_get_n_failures(const download_status_t *dls);
-int download_status_get_n_attempts(const download_status_t *dls);
-
-int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
- const char *resource);
-
-#ifdef DIRECTORY_PRIVATE
-
-/** A structure to hold arguments passed into each directory response
- * handler */
-typedef struct response_handler_args_t {
- int status_code;
- const char *reason;
- const char *body;
- size_t body_len;
- const char *headers;
-} response_handler_args_t;
-
-struct directory_request_t {
- /**
- * These fields specify which directory we're contacting. Routerstatus,
- * if present, overrides the other fields.
- *
- * @{ */
- tor_addr_port_t or_addr_port;
- tor_addr_port_t dir_addr_port;
- char digest[DIGEST_LEN];
-
- const routerstatus_t *routerstatus;
- /** @} */
- /** One of DIR_PURPOSE_* other than DIR_PURPOSE_SERVER. Describes what
- * kind of operation we'll be doing (upload/download), and of what kind
- * of document. */
- uint8_t dir_purpose;
- /** One of ROUTER_PURPOSE_*; used for uploads and downloads of routerinfo
- * and extrainfo docs. */
- uint8_t router_purpose;
- /** Enum: determines whether to anonymize, and whether to use dirport or
- * orport. */
- dir_indirection_t indirection;
- /** Alias to the variable part of the URL for this request */
- const char *resource;
- /** Alias to the payload to upload (if any) */
- const char *payload;
- /** Number of bytes to upload from payload</b> */
- size_t payload_len;
- /** Value to send in an if-modified-since header, or 0 for none. */
- time_t if_modified_since;
- /** Hidden-service-specific information v2. */
- const rend_data_t *rend_query;
- /** Extra headers to append to the request */
- struct config_line_t *additional_headers;
- /** Hidden-service-specific information for v3+. */
- const hs_ident_dir_conn_t *hs_ident;
- /** Used internally to directory.c: gets informed when the attempt to
- * connect to the directory succeeds or fails, if that attempt bears on the
- * directory's usability as a directory guard. */
- struct circuit_guard_state_t *guard_state;
-};
-
-struct get_handler_args_t;
-STATIC int handle_get_hs_descriptor_v3(dir_connection_t *conn,
- const struct get_handler_args_t *args);
-STATIC int directory_handle_command(dir_connection_t *conn);
-STATIC char *accept_encoding_header(void);
-STATIC int allowed_anonymous_connection_compression_method(
- enum compress_method_t);
-STATIC void warn_disallowed_anonymous_compression_method(
- enum compress_method_t);
-
-STATIC int handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
- const response_handler_args_t *args);
-STATIC int handle_response_fetch_microdesc(dir_connection_t *conn,
- const response_handler_args_t *args);
-
-STATIC int handle_response_fetch_consensus(dir_connection_t *conn,
- const response_handler_args_t *args);
-
-#endif /* defined(DIRECTORY_PRIVATE) */
-
-#ifdef TOR_UNIT_TESTS
-/* Used only by test_dir.c and test_hs_cache.c */
-
-STATIC int parse_http_url(const char *headers, char **url);
-STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
- const char *resource);
-MOCK_DECL(STATIC int, directory_handle_command_get,(dir_connection_t *conn,
- const char *headers,
- const char *req_body,
- size_t req_body_len));
-MOCK_DECL(STATIC int, directory_handle_command_post,(dir_connection_t *conn,
- const char *headers,
- const char *body,
- size_t body_len));
-STATIC int download_status_schedule_get_delay(download_status_t *dls,
- int min_delay,
- time_t now);
-
-STATIC int handle_post_hs_descriptor(const char *url, const char *body);
-
-STATIC char* authdir_type_to_string(dirinfo_type_t auth);
-STATIC const char * dir_conn_purpose_to_string(int purpose);
-STATIC int should_use_directory_guards(const or_options_t *options);
-enum compression_level_t;
-STATIC enum compression_level_t choose_compression_level(ssize_t n_bytes);
-STATIC int find_dl_min_delay(const download_status_t *dls,
- const or_options_t *options);
-
-STATIC int next_random_exponential_delay(int delay,
- int base_delay);
-
-STATIC void next_random_exponential_delay_range(int *low_bound_out,
- int *high_bound_out,
- int delay,
- int base_delay);
-
-STATIC int parse_hs_version_from_post(const char *url, const char *prefix,
- const char **end_pos);
-
-STATIC unsigned parse_accept_encoding_header(const char *h);
-#endif /* defined(TOR_UNIT_TESTS) */
-
-#if defined(TOR_UNIT_TESTS) || defined(DIRECTORY_PRIVATE)
-/* Used only by directory.c and test_dir.c */
-
-/* no more than quadruple the previous delay (multiplier + 1) */
-#define DIR_DEFAULT_RANDOM_MULTIPLIER (3)
-/* no more than triple the previous delay */
-#define DIR_TEST_NET_RANDOM_MULTIPLIER (2)
-
-#endif /* defined(TOR_UNIT_TESTS) || defined(DIRECTORY_PRIVATE) */
-
-#endif /* !defined(TOR_DIRECTORY_H) */
diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c
index 2fd2572a2..65c3d6d91 100644
--- a/src/feature/dircache/dirserv.c
+++ b/src/feature/dircache/dirserv.c
@@ -10,7 +10,7 @@
#include "core/mainloop/connection.h"
#include "feature/dircache/conscache.h"
#include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "feature/dircache/dirserv.h"
#include "feature/nodelist/microdesc.h"
#include "feature/nodelist/routerlist.h"
diff --git a/src/feature/dircache/directory.c b/src/feature/dirclient/dirclient.c
similarity index 54%
rename from src/feature/dircache/directory.c
rename to src/feature/dirclient/dirclient.c
index 7066e92c2..0f5f87031 100644
--- a/src/feature/dircache/directory.c
+++ b/src/feature/dirclient/dirclient.c
@@ -3,131 +3,68 @@
* Copyright (c) 2007-2018, The Tor Project, Inc. */
/* See LICENSE for licensing information */
-#define DIRECTORY_PRIVATE
+#define DIRCLIENT_PRIVATE
#include "core/or/or.h"
-#include "lib/err/backtrace.h"
-#include "feature/client/bridges.h"
-#include "lib/container/buffers.h"
-#include "core/or/circuitbuild.h"
+
#include "app/config/config.h"
#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
#include "core/or/connection_edge.h"
-#include "feature/dircache/conscache.h"
-#include "feature/dircommon/consdiff.h"
-#include "feature/dircache/consdiffmgr.h"
+#include "core/or/policies.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
#include "feature/control/control.h"
-#include "lib/compress/compress.h"
-#include "lib/crypt_ops/crypto_rand.h"
-#include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/mode.h"
+#include "feature/dirauth/shared_random.h"
#include "feature/dircache/dirserv.h"
-#include "feature/dirauth/process_descs.h"
-#include "feature/client/entrynodes.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/consdiff.h"
+#include "feature/dircommon/directory.h"
#include "feature/dircommon/fp_pair.h"
-#include "feature/stats/geoip.h"
#include "feature/hs/hs_cache.h"
-#include "feature/hs/hs_common.h"
-#include "feature/hs/hs_control.h"
#include "feature/hs/hs_client.h"
-#include "core/mainloop/mainloop.h"
+#include "feature/hs/hs_control.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/microdesc.h"
#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/node_select.h"
#include "feature/nodelist/nodelist.h"
-#include "core/or/policies.h"
-#include "core/or/relay.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerset.h"
+#include "feature/relay/router.h"
+#include "feature/rend/rendcache.h"
#include "feature/rend/rendclient.h"
#include "feature/rend/rendcommon.h"
#include "feature/rend/rendservice.h"
#include "feature/stats/rephist.h"
-#include "feature/relay/router.h"
-#include "feature/nodelist/authcert.h"
-#include "feature/nodelist/dirlist.h"
-#include "feature/nodelist/node_select.h"
-#include "feature/nodelist/routerlist.h"
-#include "feature/nodelist/routerparse.h"
-#include "feature/nodelist/routerset.h"
-#include "lib/encoding/confline.h"
+#include "lib/compress/compress.h"
#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/err/backtrace.h"
-#if defined(EXPORTMALLINFO) && defined(HAVE_MALLOC_H) && defined(HAVE_MALLINFO)
-#if !defined(OpenBSD)
-#include <malloc.h>
-#endif
-#endif
-
-#include "feature/dirauth/dirvote.h"
-#include "feature/dirauth/mode.h"
-#include "feature/dirauth/shared_random.h"
-
-#include "feature/nodelist/authority_cert_st.h"
+#include "core/or/entry_connection_st.h"
#include "feature/dircache/cached_dir_st.h"
-#include "feature/dircommon/dir_connection_st.h"
#include "feature/dirclient/dir_server_st.h"
-#include "core/or/entry_connection_st.h"
+#include "feature/dircommon/dir_connection_st.h"
#include "feature/nodelist/networkstatus_st.h"
#include "feature/nodelist/node_st.h"
-#include "feature/rend/rend_service_descriptor_st.h"
#include "feature/nodelist/routerinfo_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
-/**
- * \file directory.c
- * \brief Code to send and fetch information from directory authorities and
- * caches via HTTP.
- *
- * Directory caches and authorities use dirserv.c to generate the results of a
- * query and stream them to the connection; clients use routerparse.c to parse
- * them.
- *
- * Every directory request has a dir_connection_t on the client side and on
- * the server side. In most cases, the dir_connection_t object is a linked
- * connection, tunneled through an edge_connection_t so that it can be a
- * stream on the Tor network. The only non-tunneled connections are those
- * that are used to upload material (descriptors and votes) to authorities.
- * Among tunneled connections, some use one-hop circuits, and others use
- * multi-hop circuits for anonymity.
- *
- * Directory requests are launched by calling
- * directory_initiate_request(). This
- * launch the connection, will construct an HTTP request with
- * directory_send_command(), send the and wait for a response. The client
- * later handles the response with connection_dir_client_reached_eof(),
- * which passes the information received to another part of Tor.
- *
- * On the server side, requests are read in directory_handle_command(),
- * which dispatches first on the request type (GET or POST), and then on
- * the URL requested. GET requests are processed with a table-based
- * dispatcher in url_table[]. The process of handling larger GET requests
- * is complicated because we need to avoid allocating a copy of all the
- * data to be sent to the client in one huge buffer. Instead, we spool the
- * data into the buffer using logic in connection_dirserv_flushed_some() in
- * dirserv.c. (TODO: If we extended buf.c to have a zero-copy
- * reference-based buffer type, we could remove most of that code, at the
- * cost of a bit more reference counting.)
- **/
+/** Maximum size, in bytes, for any directory object that we've downloaded. */
+#define MAX_DIR_DL_SIZE ((1<<24)-1) /* 16 MB - 1 */
+
+/** How far in the future do we allow a directory server to tell us it is
+ * before deciding that one of us has the wrong time? */
+#define ALLOW_DIRECTORY_TIME_SKEW (30*60)
-/* In-points to directory.c:
- *
- * - directory_post_to_dirservers(), called from
- * router_upload_dir_desc_to_dirservers() in router.c
- * upload_service_descriptor() in rendservice.c
- * - directory_get_from_dirserver(), called from
- * rend_client_refetch_renddesc() in rendclient.c
- * run_scheduled_events() in main.c
- * do_hup() in main.c
- * - connection_dir_process_inbuf(), called from
- * connection_process_inbuf() in connection.c
- * - connection_dir_finished_flushing(), called from
- * connection_finished_flushing() in connection.c
- * - connection_dir_finished_connecting(), called from
- * connection_finished_connecting() in connection.c
- */
-static void directory_send_command(dir_connection_t *conn,
- int direct,
- const directory_request_t *request);
static int body_is_plausible(const char *body, size_t body_len, int purpose);
-static void http_set_address_origin(const char *headers, connection_t *conn);
static void connection_dir_download_routerdesc_failed(dir_connection_t *conn);
static void connection_dir_bridge_routerdesc_failed(dir_connection_t *conn);
static void connection_dir_download_cert_failed(
@@ -141,126 +78,12 @@ static void dir_routerdesc_download_failed(smartlist_t *failed,
static void dir_microdesc_download_failed(smartlist_t *failed,
int status_code,
const char *dir_id);
-static int client_likes_consensus(const struct consensus_cache_entry_t *ent,
- const char *want_url);
-
+static void directory_send_command(dir_connection_t *conn,
+ const int direct,
+ const directory_request_t *req);
static void connection_dir_close_consensus_fetches(
dir_connection_t *except_this_one, const char *resource);
-/********* START VARIABLES **********/
-
-/** Maximum size, in bytes, for resized buffers. */
-#define MAX_BUF_SIZE ((1<<24)-1) /* 16MB-1 */
-/** Maximum size, in bytes, for any directory object that we've downloaded. */
-#define MAX_DIR_DL_SIZE MAX_BUF_SIZE
-
-/** Maximum size, in bytes, for any directory object that we're accepting
- * as an upload. */
-#define MAX_DIR_UL_SIZE MAX_BUF_SIZE
-
-/** How far in the future do we allow a directory server to tell us it is
- * before deciding that one of us has the wrong time? */
-#define ALLOW_DIRECTORY_TIME_SKEW (30*60)
-
-#define X_ADDRESS_HEADER "X-Your-Address-Is: "
-#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: "
-
-/** HTTP cache control: how long do we tell proxies they can cache each
- * kind of document we serve? */
-#define FULL_DIR_CACHE_LIFETIME (60*60)
-#define RUNNINGROUTERS_CACHE_LIFETIME (20*60)
-#define DIRPORTFRONTPAGE_CACHE_LIFETIME (20*60)
-#define NETWORKSTATUS_CACHE_LIFETIME (5*60)
-#define ROUTERDESC_CACHE_LIFETIME (30*60)
-#define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
-#define ROBOTS_CACHE_LIFETIME (24*60*60)
-#define MICRODESC_CACHE_LIFETIME (48*60*60)
-
-/********* END VARIABLES ************/
-
-/** Convert a connection_t* to a dir_connection_t*; assert if the cast is
- * invalid. */
-dir_connection_t *
-TO_DIR_CONN(connection_t *c)
-{
- tor_assert(c->magic == DIR_CONNECTION_MAGIC);
- return DOWNCAST(dir_connection_t, c);
-}
-
-/** Return false if the directory purpose <b>dir_purpose</b>
- * does not require an anonymous (three-hop) connection.
- *
- * Return true 1) by default, 2) if all directory actions have
- * specifically been configured to be over an anonymous connection,
- * or 3) if the router is a bridge */
-int
-purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
- const char *resource)
-{
- if (get_options()->AllDirActionsPrivate)
- return 1;
-
- if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
- if (dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC
- && resource && !strcmp(resource, "authority.z")) {
- /* We are asking a bridge for its own descriptor. That doesn't need
- anonymity. */
- return 0;
- }
- /* Assume all other bridge stuff needs anonymity. */
- return 1; /* if no circuits yet, this might break bootstrapping, but it's
- * needed to be safe. */
- }
-
- switch (dir_purpose)
- {
- case DIR_PURPOSE_UPLOAD_DIR:
- case DIR_PURPOSE_UPLOAD_VOTE:
- case DIR_PURPOSE_UPLOAD_SIGNATURES:
- case DIR_PURPOSE_FETCH_STATUS_VOTE:
- case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
- case DIR_PURPOSE_FETCH_CONSENSUS:
- case DIR_PURPOSE_FETCH_CERTIFICATE:
- case DIR_PURPOSE_FETCH_SERVERDESC:
- case DIR_PURPOSE_FETCH_EXTRAINFO:
- case DIR_PURPOSE_FETCH_MICRODESC:
- return 0;
- case DIR_PURPOSE_HAS_FETCHED_HSDESC:
- case DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2:
- case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
- case DIR_PURPOSE_FETCH_RENDDESC_V2:
- case DIR_PURPOSE_FETCH_HSDESC:
- case DIR_PURPOSE_UPLOAD_HSDESC:
- return 1;
- case DIR_PURPOSE_SERVER:
- default:
- log_warn(LD_BUG, "Called with dir_purpose=%d, router_purpose=%d",
- dir_purpose, router_purpose);
- tor_assert_nonfatal_unreached();
- return 1; /* Assume it needs anonymity; better safe than sorry. */
- }
-}
-
-/** Return a newly allocated string describing <b>auth</b>. Only describes
- * authority features. */
-STATIC char *
-authdir_type_to_string(dirinfo_type_t auth)
-{
- char *result;
- smartlist_t *lst = smartlist_new();
- if (auth & V3_DIRINFO)
- smartlist_add(lst, (void*)"V3");
- if (auth & BRIDGE_DIRINFO)
- smartlist_add(lst, (void*)"Bridge");
- if (smartlist_len(lst)) {
- result = smartlist_join_strings(lst, ", ", 0, NULL);
- } else {
- result = tor_strdup("[Not an authority]");
- }
- smartlist_free(lst);
- return result;
-}
-
/** Return a string describing a given directory connection purpose. */
STATIC const char *
dir_conn_purpose_to_string(int purpose)
@@ -606,7 +429,6 @@ dir_consensus_request_set_additional_headers(directory_request_t *req,
directory_request_add_header(req, X_OR_DIFF_FROM_CONSENSUS_HEADER, hex);
}
}
-
/** Start a connection to a random running directory server, using
* connection purpose <b>dir_purpose</b>, intending to fetch descriptors
* of purpose <b>router_purpose</b>, and requesting <b>resource</b>.
@@ -890,8 +712,8 @@ directory_conn_is_self_reachability_test(dir_connection_t *conn)
* server due to a network error: Mark the router as down and try again if
* possible.
*/
-static void
-connection_dir_request_failed(dir_connection_t *conn)
+void
+connection_dir_client_request_failed(dir_connection_t *conn)
{
if (conn->guard_state) {
/* We haven't seen a success on this guard state, so consider it to have
@@ -1608,22 +1430,11 @@ directory_initiate_request,(directory_request_t *request))
}
}
-/** Return true iff anything we say on <b>conn</b> is being encrypted before
- * we send it to the client/server. */
-int
-connection_dir_is_encrypted(const dir_connection_t *conn)
-{
- /* Right now it's sufficient to see if conn is or has been linked, since
- * the only thing it could be linked to is an edge connection on a
- * circuit, and the only way it could have been unlinked is at the edge
- * connection getting closed.
- */
- return TO_CONN(conn)->linked;
-}
-
/** Helper for sorting
*
* sort strings alphabetically
+ *
+ * XXXX we have a smartlist_sort_strings() function, right?
*/
static int
compare_strs_(const void **a, const void **b)
@@ -1952,226 +1763,6 @@ directory_send_command(dir_connection_t *conn,
(payload ? payload_len : 0));
}
-/** Parse an HTTP request string <b>headers</b> of the form
- * \verbatim
- * "\%s [http[s]://]\%s HTTP/1..."
- * \endverbatim
- * If it's well-formed, strdup the second \%s into *<b>url</b>, and
- * nul-terminate it. If the url doesn't start with "/tor/", rewrite it
- * so it does. Return 0.
- * Otherwise, return -1.
- */
-STATIC int
-parse_http_url(const char *headers, char **url)
-{
- char *command = NULL;
- if (parse_http_command(headers, &command, url) < 0) {
- return -1;
- }
- if (strcmpstart(*url, "/tor/")) {
- char *new_url = NULL;
- tor_asprintf(&new_url, "/tor%s%s",
- *url[0] == '/' ? "" : "/",
- *url);
- tor_free(*url);
- *url = new_url;
- }
- tor_free(command);
- return 0;
-}
-
-/** Parse an HTTP request line at the start of a headers string. On failure,
- * return -1. On success, set *<b>command_out</b> to a copy of the HTTP
- * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and
- * return 0. */
-int
-parse_http_command(const char *headers, char **command_out, char **url_out)
-{
- const char *command, *end_of_command;
- char *s, *start, *tmp;
-
- s = (char *)eat_whitespace_no_nl(headers);
- if (!*s) return -1;
- command = s;
- s = (char *)find_whitespace(s); /* get past GET/POST */
- if (!*s) return -1;
- end_of_command = s;
- s = (char *)eat_whitespace_no_nl(s);
- if (!*s) return -1;
- start = s; /* this is the URL, assuming it's valid */
- s = (char *)find_whitespace(start);
- if (!*s) return -1;
-
- /* tolerate the http[s] proxy style of putting the hostname in the url */
- if (s-start >= 4 && !strcmpstart(start,"http")) {
- tmp = start + 4;
- if (*tmp == 's')
- tmp++;
- if (s-tmp >= 3 && !strcmpstart(tmp,"://")) {
- tmp = strchr(tmp+3, '/');
- if (tmp && tmp < s) {
- log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string");
- start = tmp;
- }
- }
- }
-
- /* Check if the header is well formed (next sequence
- * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */
- {
- unsigned minor_ver;
- char ch;
- char *e = (char *)eat_whitespace_no_nl(s);
- if (2 != tor_sscanf(e, "HTTP/1.%u%c", &minor_ver, &ch)) {
- return -1;
- }
- if (ch != '\r')
- return -1;
- }
-
- *url_out = tor_memdup_nulterm(start, s-start);
- *command_out = tor_memdup_nulterm(command, end_of_command - command);
- return 0;
-}
-
-/** Return a copy of the first HTTP header in <b>headers</b> whose key is
- * <b>which</b>. The key should be given with a terminating colon and space;
- * this function copies everything after, up to but not including the
- * following \\r\\n. */
-char *
-http_get_header(const char *headers, const char *which)
-{
- const char *cp = headers;
- while (cp) {
- if (!strcasecmpstart(cp, which)) {
- char *eos;
- cp += strlen(which);
- if ((eos = strchr(cp,'\r')))
- return tor_strndup(cp, eos-cp);
- else
- return tor_strdup(cp);
- }
- cp = strchr(cp, '\n');
- if (cp)
- ++cp;
- }
- return NULL;
-}
-
-/** If <b>headers</b> indicates that a proxy was involved, then rewrite
- * <b>conn</b>-\>address to describe our best guess of the address that
- * originated this HTTP request. */
-static void
-http_set_address_origin(const char *headers, connection_t *conn)
-{
- char *fwd;
-
- fwd = http_get_header(headers, "Forwarded-For: ");
- if (!fwd)
- fwd = http_get_header(headers, "X-Forwarded-For: ");
- if (fwd) {
- tor_addr_t toraddr;
- if (tor_addr_parse(&toraddr,fwd) == -1 ||
- tor_addr_is_internal(&toraddr,0)) {
- log_debug(LD_DIR, "Ignoring local/internal IP %s", escaped(fwd));
- tor_free(fwd);
- return;
- }
-
- tor_free(conn->address);
- conn->address = tor_strdup(fwd);
- tor_free(fwd);
- }
-}
-
-/** Parse an HTTP response string <b>headers</b> of the form
- * \verbatim
- * "HTTP/1.\%d \%d\%s\r\n...".
- * \endverbatim
- *
- * If it's well-formed, assign the status code to *<b>code</b> and
- * return 0. Otherwise, return -1.
- *
- * On success: If <b>date</b> is provided, set *date to the Date
- * header in the http headers, or 0 if no such header is found. If
- * <b>compression</b> is provided, set *<b>compression</b> to the
- * compression method given in the Content-Encoding header, or 0 if no
- * such header is found, or -1 if the value of the header is not
- * recognized. If <b>reason</b> is provided, strdup the reason string
- * into it.
- */
-int
-parse_http_response(const char *headers, int *code, time_t *date,
- compress_method_t *compression, char **reason)
-{
- unsigned n1, n2;
- char datestr[RFC1123_TIME_LEN+1];
- smartlist_t *parsed_headers;
- tor_assert(headers);
- tor_assert(code);
-
- while (TOR_ISSPACE(*headers)) headers++; /* tolerate leading whitespace */
-
- if (tor_sscanf(headers, "HTTP/1.%u %u", &n1, &n2) < 2 ||
- (n1 != 0 && n1 != 1) ||
- (n2 < 100 || n2 >= 600)) {
- log_warn(LD_HTTP,"Failed to parse header %s",escaped(headers));
- return -1;
- }
- *code = n2;
-
- parsed_headers = smartlist_new();
- smartlist_split_string(parsed_headers, headers, "\n",
- SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
- if (reason) {
- smartlist_t *status_line_elements = smartlist_new();
- tor_assert(smartlist_len(parsed_headers));
- smartlist_split_string(status_line_elements,
- smartlist_get(parsed_headers, 0),
- " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
- tor_assert(smartlist_len(status_line_elements) <= 3);
- if (smartlist_len(status_line_elements) == 3) {
- *reason = smartlist_get(status_line_elements, 2);
- smartlist_set(status_line_elements, 2, NULL); /* Prevent free */
- }
- SMARTLIST_FOREACH(status_line_elements, char *, cp, tor_free(cp));
- smartlist_free(status_line_elements);
- }
- if (date) {
- *date = 0;
- SMARTLIST_FOREACH(parsed_headers, const char *, s,
- if (!strcmpstart(s, "Date: ")) {
- strlcpy(datestr, s+6, sizeof(datestr));
- /* This will do nothing on failure, so we don't need to check
- the result. We shouldn't warn, since there are many other valid
- date formats besides the one we use. */
- parse_rfc1123_time(datestr, date);
- break;
- });
- }
- if (compression) {
- const char *enc = NULL;
- SMARTLIST_FOREACH(parsed_headers, const char *, s,
- if (!strcmpstart(s, "Content-Encoding: ")) {
- enc = s+18; break;
- });
-
- if (enc == NULL)
- *compression = NO_METHOD;
- else {
- *compression = compression_method_get_by_name(enc);
-
- if (*compression == UNKNOWN_METHOD)
- log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
- escaped(enc));
- }
- }
- SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
- smartlist_free(parsed_headers);
-
- return 0;
-}
-
/** Return true iff <b>body</b> doesn't start with a plausible router or
* network-status or microdescriptor opening. This is a sign of possible
* compression. */
@@ -3399,63 +2990,11 @@ connection_dir_reached_eof(dir_connection_t *conn)
connection_mark_for_close(TO_CONN(conn));
return retval;
}
-
-/** If any directory object is arriving, and it's over 10MB large, we're
- * getting DoS'd. (As of 0.1.2.x, raw directories are about 1MB, and we never
- * ask for more than 96 router descriptors at a time.)
- */
-#define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20))
-
-#define MAX_VOTE_DL_SIZE (MAX_DIRECTORY_OBJECT_SIZE * 5)
-
-/** Read handler for directory connections. (That's connections <em>to</em>
- * directory servers and connections <em>at</em> directory servers.)
- */
-int
-connection_dir_process_inbuf(dir_connection_t *conn)
-{
- size_t max_size;
- tor_assert(conn);
- tor_assert(conn->base_.type == CONN_TYPE_DIR);
-
- /* Directory clients write, then read data until they receive EOF;
- * directory servers read data until they get an HTTP command, then
- * write their response (when it's finished flushing, they mark for
- * close).
- */
-
- /* If we're on the dirserver side, look for a command. */
- if (conn->base_.state == DIR_CONN_STATE_SERVER_COMMAND_WAIT) {
- if (directory_handle_command(conn) < 0) {
- connection_mark_for_close(TO_CONN(conn));
- return -1;
- }
- return 0;
- }
-
- max_size =
- (TO_CONN(conn)->purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) ?
- MAX_VOTE_DL_SIZE : MAX_DIRECTORY_OBJECT_SIZE;
-
- if (connection_get_inbuf_len(TO_CONN(conn)) > max_size) {
- log_warn(LD_HTTP,
- "Too much data received from directory connection (%s): "
- "denial of service attempt, or you need to upgrade?",
- conn->base_.address);
- connection_mark_for_close(TO_CONN(conn));
- return -1;
- }
-
- if (!conn->base_.inbuf_reached_eof)
- log_debug(LD_HTTP,"Got data, not eof. Leaving on inbuf.");
- return 0;
-}
-
/** We are closing a dir connection: If <b>dir_conn</b> is a dir connection
* that tried to fetch an HS descriptor, check if it successfully fetched it,
* or if we need to try again. */
-static void
-refetch_hsdesc_if_needed(dir_connection_t *dir_conn)
+void
+connection_dir_client_refetch_hsdesc_if_needed(dir_connection_t *dir_conn)
{
connection_t *conn = TO_CONN(dir_conn);
@@ -3478,1776 +3017,78 @@ refetch_hsdesc_if_needed(dir_connection_t *dir_conn)
}
}
-/** Called when we're about to finally unlink and free a directory connection:
- * perform necessary accounting and cleanup */
-void
-connection_dir_about_to_close(dir_connection_t *dir_conn)
-{
- connection_t *conn = TO_CONN(dir_conn);
-
- if (conn->state < DIR_CONN_STATE_CLIENT_FINISHED) {
- /* It's a directory connection and connecting or fetching
- * failed: forget about this router, and maybe try again. */
- connection_dir_request_failed(dir_conn);
- }
+/** Array of compression methods to use (if supported) for requesting
+ * compressed data, ordered from best to worst. */
+static compress_method_t client_meth_pref[] = {
+ LZMA_METHOD,
+ ZSTD_METHOD,
+ ZLIB_METHOD,
+ GZIP_METHOD,
+ NO_METHOD
+};
- refetch_hsdesc_if_needed(dir_conn);
-}
+/** Array of allowed compression methods to use (if supported) when receiving a
+ * response from a request that was required to be anonymous. */
+static compress_method_t client_meth_allowed_anonymous_compression[] = {
+ ZLIB_METHOD,
+ GZIP_METHOD,
+ NO_METHOD
+};
-/** Create an http response for the client <b>conn</b> out of
- * <b>status</b> and <b>reason_phrase</b>. Write it to <b>conn</b>.
- */
-static void
-write_short_http_response(dir_connection_t *conn, int status,
- const char *reason_phrase)
+/** Return a newly allocated string containing a comma separated list of
+ * supported encodings. */
+STATIC char *
+accept_encoding_header(void)
{
- char *buf = NULL;
- char *datestring = NULL;
-
- IF_BUG_ONCE(!reason_phrase) { /* bullet-proofing */
- reason_phrase = "unspecified";
- }
+ smartlist_t *methods = smartlist_new();
+ char *header = NULL;
+ compress_method_t method;
+ unsigned i;
- if (server_mode(get_options())) {
- /* include the Date: header, but only if we're a relay or bridge */
- char datebuf[RFC1123_TIME_LEN+1];
- format_rfc1123_time(datebuf, time(NULL));
- tor_asprintf(&datestring, "Date: %s\r\n", datebuf);
+ for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) {
+ method = client_meth_pref[i];
+ if (tor_compress_supports_method(method))
+ smartlist_add(methods, (char *)compression_method_get_name(method));
}
- tor_asprintf(&buf, "HTTP/1.0 %d %s\r\n%s\r\n",
- status, reason_phrase, datestring?datestring:"");
-
- log_debug(LD_DIRSERV,"Wrote status 'HTTP/1.0 %d %s'", status, reason_phrase);
- connection_buf_add(buf, strlen(buf), TO_CONN(conn));
+ header = smartlist_join_strings(methods, ", ", 0, NULL);
+ smartlist_free(methods);
- tor_free(datestring);
- tor_free(buf);
+ return header;
}
-/** Write the header for an HTTP/1.0 response onto <b>conn</b>-\>outbuf,
- * with <b>type</b> as the Content-Type.
- *
- * If <b>length</b> is nonnegative, it is the Content-Length.
- * If <b>encoding</b> is provided, it is the Content-Encoding.
- * If <b>cache_lifetime</b> is greater than 0, the content may be cached for
- * up to cache_lifetime seconds. Otherwise, the content may not be cached. */
-static void
-write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
- const char *type, const char *encoding,
- const char *extra_headers,
- long cache_lifetime)
+/** Check if the given compression method is allowed for a connection that is
+ * supposed to be anonymous. Returns 1 if the compression method is allowed,
+ * otherwise 0. */
+STATIC int
+allowed_anonymous_connection_compression_method(compress_method_t method)
{
- char date[RFC1123_TIME_LEN+1];
- time_t now = time(NULL);
- buf_t *buf = buf_new_with_capacity(1024);
-
- tor_assert(conn);
-
- format_rfc1123_time(date, now);
-
- buf_add_printf(buf, "HTTP/1.0 200 OK\r\nDate: %s\r\n", date);
- if (type) {
- buf_add_printf(buf, "Content-Type: %s\r\n", type);
- }
- if (!is_local_addr(&conn->base_.addr)) {
- /* Don't report the source address for a nearby/private connection.
- * Otherwise we tend to mis-report in cases where incoming ports are
- * being forwarded to a Tor server running behind the firewall. */
- buf_add_printf(buf, X_ADDRESS_HEADER "%s\r\n", conn->base_.address);
- }
- if (encoding) {
- buf_add_printf(buf, "Content-Encoding: %s\r\n", encoding);
- }
- if (length >= 0) {
- buf_add_printf(buf, "Content-Length: %ld\r\n", (long)length);
- }
- if (cache_lifetime > 0) {
- char expbuf[RFC1123_TIME_LEN+1];
- format_rfc1123_time(expbuf, (time_t)(now + cache_lifetime));
- /* We could say 'Cache-control: max-age=%d' here if we start doing
- * http/1.1 */
- buf_add_printf(buf, "Expires: %s\r\n", expbuf);
- } else if (cache_lifetime == 0) {
- /* We could say 'Cache-control: no-cache' here if we start doing
- * http/1.1 */
- buf_add_string(buf, "Pragma: no-cache\r\n");
- }
- if (extra_headers) {
- buf_add_string(buf, extra_headers);
- }
- buf_add_string(buf, "\r\n");
-
- connection_buf_add_buf(TO_CONN(conn), buf);
- buf_free(buf);
-}
-
-/** As write_http_response_header_impl, but sets encoding and content-typed
- * based on whether the response will be <b>compressed</b> or not. */
-static void
-write_http_response_headers(dir_connection_t *conn, ssize_t length,
- compress_method_t method,
- const char *extra_headers, long cache_lifetime)
-{
- const char *methodname = compression_method_get_name(method);
- const char *doctype;
- if (method == NO_METHOD)
- doctype = "text/plain";
- else
- doctype = "application/octet-stream";
- write_http_response_header_impl(conn, length,
- doctype,
- methodname,
- extra_headers,
- cache_lifetime);
-}
-
-/** As write_http_response_headers, but assumes extra_headers is NULL */
-static void
-write_http_response_header(dir_connection_t *conn, ssize_t length,
- compress_method_t method,
- long cache_lifetime)
-{
- write_http_response_headers(conn, length, method, NULL, cache_lifetime);
-}
-
-/** Array of compression methods to use (if supported) for serving
- * precompressed data, ordered from best to worst. */
-static compress_method_t srv_meth_pref_precompressed[] = {
- LZMA_METHOD,
- ZSTD_METHOD,
- ZLIB_METHOD,
- GZIP_METHOD,
- NO_METHOD
-};
-
-/** Array of compression methods to use (if supported) for serving
- * streamed data, ordered from best to worst. */
-static compress_method_t srv_meth_pref_streaming_compression[] = {
- ZSTD_METHOD,
- ZLIB_METHOD,
- GZIP_METHOD,
- NO_METHOD
-};
-
-/** Array of allowed compression methods to use (if supported) when receiving a
- * response from a request that was required to be anonymous. */
-static compress_method_t client_meth_allowed_anonymous_compression[] = {
- ZLIB_METHOD,
- GZIP_METHOD,
- NO_METHOD
-};
-
-/** Parse the compression methods listed in an Accept-Encoding header <b>h</b>,
- * and convert them to a bitfield where compression method x is supported if
- * and only if 1 << x is set in the bitfield. */
-STATIC unsigned
-parse_accept_encoding_header(const char *h)
-{
- unsigned result = (1u << NO_METHOD);
- smartlist_t *methods = smartlist_new();
- smartlist_split_string(methods, h, ",",
- SPLIT_SKIP_SPACE|SPLIT_STRIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-
- SMARTLIST_FOREACH_BEGIN(methods, const char *, m) {
- compress_method_t method = compression_method_get_by_name(m);
- if (method != UNKNOWN_METHOD) {
- tor_assert(((unsigned)method) < 8*sizeof(unsigned));
- result |= (1u << method);
- }
- } SMARTLIST_FOREACH_END(m);
- SMARTLIST_FOREACH_BEGIN(methods, char *, m) {
- tor_free(m);
- } SMARTLIST_FOREACH_END(m);
- smartlist_free(methods);
- return result;
-}
-
-/** Array of compression methods to use (if supported) for requesting
- * compressed data, ordered from best to worst. */
-static compress_method_t client_meth_pref[] = {
- LZMA_METHOD,
- ZSTD_METHOD,
- ZLIB_METHOD,
- GZIP_METHOD,
- NO_METHOD
-};
-
-/** Return a newly allocated string containing a comma separated list of
- * supported encodings. */
-STATIC char *
-accept_encoding_header(void)
-{
- smartlist_t *methods = smartlist_new();
- char *header = NULL;
- compress_method_t method;
- unsigned i;
-
- for (i = 0; i < ARRAY_LENGTH(client_meth_pref); ++i) {
- method = client_meth_pref[i];
- if (tor_compress_supports_method(method))
- smartlist_add(methods, (char *)compression_method_get_name(method));
- }
-
- header = smartlist_join_strings(methods, ", ", 0, NULL);
- smartlist_free(methods);
-
- return header;
-}
-
-/** Decide whether a client would accept the consensus we have.
- *
- * Clients can say they only want a consensus if it's signed by more
- * than half the authorities in a list. They pass this list in
- * the url as "...consensus/<b>fpr</b>+<b>fpr</b>+<b>fpr</b>".
- *
- * <b>fpr</b> may be an abbreviated fingerprint, i.e. only a left substring
- * of the full authority identity digest. (Only strings of even length,
- * i.e. encodings of full bytes, are handled correctly. In the case
- * of an odd number of hex digits the last one is silently ignored.)
- *
- * Returns 1 if more than half of the requested authorities signed the
- * consensus, 0 otherwise.
- */
-int
-client_likes_consensus(const struct consensus_cache_entry_t *ent,
- const char *want_url)
-{
- smartlist_t *voters = smartlist_new();
- int need_at_least;
- int have = 0;
-
- if (consensus_cache_entry_get_voter_id_digests(ent, voters) != 0) {
- smartlist_free(voters);
- return 1; // We don't know the voters; assume the client won't mind. */
- }
-
- smartlist_t *want_authorities = smartlist_new();
- dir_split_resource_into_fingerprints(want_url, want_authorities, NULL, 0);
- need_at_least = smartlist_len(want_authorities)/2+1;
-
- SMARTLIST_FOREACH_BEGIN(want_authorities, const char *, want_digest) {
-
- SMARTLIST_FOREACH_BEGIN(voters, const char *, digest) {
- if (!strcasecmpstart(digest, want_digest)) {
- have++;
- break;
- };
- } SMARTLIST_FOREACH_END(digest);
-
- /* early exit, if we already have enough */
- if (have >= need_at_least)
- break;
- } SMARTLIST_FOREACH_END(want_digest);
-
- SMARTLIST_FOREACH(want_authorities, char *, d, tor_free(d));
- smartlist_free(want_authorities);
- SMARTLIST_FOREACH(voters, char *, cp, tor_free(cp));
- smartlist_free(voters);
- return (have >= need_at_least);
-}
-
-/** Return the compression level we should use for sending a compressed
- * response of size <b>n_bytes</b>. */
-STATIC compression_level_t
-choose_compression_level(ssize_t n_bytes)
-{
- if (! have_been_under_memory_pressure()) {
- return HIGH_COMPRESSION; /* we have plenty of RAM. */
- } else if (n_bytes < 0) {
- return HIGH_COMPRESSION; /* unknown; might be big. */
- } else if (n_bytes < 1024) {
- return LOW_COMPRESSION;
- } else if (n_bytes < 2048) {
- return MEDIUM_COMPRESSION;
- } else {
- return HIGH_COMPRESSION;
- }
-}
-
-/** Information passed to handle a GET request. */
-typedef struct get_handler_args_t {
- /** Bitmask of compression methods that the client said (or implied) it
- * supported. */
- unsigned compression_supported;
- /** If nonzero, the time included an if-modified-since header with this
- * value. */
- time_t if_modified_since;
- /** String containing the requested URL or resource. */
- const char *url;
- /** String containing the HTTP headers */
- const char *headers;
-} get_handler_args_t;
-
-/** Entry for handling an HTTP GET request.
- *
- * This entry matches a request if "string" is equal to the requested
- * resource, or if "is_prefix" is true and "string" is a prefix of the
- * requested resource.
- *
- * The 'handler' function is called to handle the request. It receives
- * an arguments structure, and must return 0 on success or -1 if we should
- * close the connection.
- **/
-typedef struct url_table_ent_s {
- const char *string;
- int is_prefix;
- int (*handler)(dir_connection_t *conn, const get_handler_args_t *args);
-} url_table_ent_t;
-
-static int handle_get_frontpage(dir_connection_t *conn,
- const get_handler_args_t *args);
-static int handle_get_current_consensus(dir_connection_t *conn,
- const get_handler_args_t *args);
-static int handle_get_status_vote(dir_connection_t *conn,
- const get_handler_args_t *args);
-static int handle_get_microdesc(dir_connection_t *conn,
- const get_handler_args_t *args);
-static int handle_get_descriptor(dir_connection_t *conn,
- const get_handler_args_t *args);
-static int handle_get_keys(dir_connection_t *conn,
- const get_handler_args_t *args);
-static int handle_get_hs_descriptor_v2(dir_connection_t *conn,
- const get_handler_args_t *args);
-static int handle_get_robots(dir_connection_t *conn,
- const get_handler_args_t *args);
-static int handle_get_networkstatus_bridges(dir_connection_t *conn,
- const get_handler_args_t *args);
-
-/** Table for handling GET requests. */
-static const url_table_ent_t url_table[] = {
- { "/tor/", 0, handle_get_frontpage },
- { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
- { "/tor/status-vote/current/", 1, handle_get_status_vote },
- { "/tor/status-vote/next/", 1, handle_get_status_vote },
- { "/tor/micro/d/", 1, handle_get_microdesc },
- { "/tor/server/", 1, handle_get_descriptor },
- { "/tor/extra/", 1, handle_get_descriptor },
- { "/tor/keys/", 1, handle_get_keys },
- { "/tor/rendezvous2/", 1, handle_get_hs_descriptor_v2 },
- { "/tor/hs/3/", 1, handle_get_hs_descriptor_v3 },
- { "/tor/robots.txt", 0, handle_get_robots },
- { "/tor/networkstatus-bridges", 0, handle_get_networkstatus_bridges },
- { NULL, 0, NULL },
-};
-
-/** Helper function: called when a dirserver gets a complete HTTP GET
- * request. Look for a request for a directory or for a rendezvous
- * service descriptor. On finding one, write a response into
- * conn-\>outbuf. If the request is unrecognized, send a 404.
- * Return 0 if we handled this successfully, or -1 if we need to close
- * the connection. */
-MOCK_IMPL(STATIC int,
-directory_handle_command_get,(dir_connection_t *conn, const char *headers,
- const char *req_body, size_t req_body_len))
-{
- char *url, *url_mem, *header;
- time_t if_modified_since = 0;
- int zlib_compressed_in_url;
- unsigned compression_methods_supported;
-
- /* We ignore the body of a GET request. */
- (void)req_body;
- (void)req_body_len;
-
- log_debug(LD_DIRSERV,"Received GET command.");
-
- conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
-
- if (parse_http_url(headers, &url) < 0) {
- write_short_http_response(conn, 400, "Bad request");
- return 0;
- }
- if ((header = http_get_header(headers, "If-Modified-Since: "))) {
- struct tm tm;
- if (parse_http_time(header, &tm) == 0) {
- if (tor_timegm(&tm, &if_modified_since)<0) {
- if_modified_since = 0;
- } else {
- log_debug(LD_DIRSERV, "If-Modified-Since is '%s'.", escaped(header));
- }
- }
- /* The correct behavior on a malformed If-Modified-Since header is to
- * act as if no If-Modified-Since header had been given. */
- tor_free(header);
- }
- log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
-
- url_mem = url;
- {
- size_t url_len = strlen(url);
-
- zlib_compressed_in_url = url_len > 2 && !strcmp(url+url_len-2, ".z");
- if (zlib_compressed_in_url) {
- url[url_len-2] = '\0';
- }
- }
-
- if ((header = http_get_header(headers, "Accept-Encoding: "))) {
- compression_methods_supported = parse_accept_encoding_header(header);
- tor_free(header);
- } else {
- compression_methods_supported = (1u << NO_METHOD);
- }
- if (zlib_compressed_in_url) {
- compression_methods_supported |= (1u << ZLIB_METHOD);
- }
-
- /* Remove all methods that we don't both support. */
- compression_methods_supported &= tor_compress_get_supported_method_bitmask();
-
- get_handler_args_t args;
- args.url = url;
- args.headers = headers;
- args.if_modified_since = if_modified_since;
- args.compression_supported = compression_methods_supported;
-
- int i, result = -1;
- for (i = 0; url_table[i].string; ++i) {
- int match;
- if (url_table[i].is_prefix) {
- match = !strcmpstart(url, url_table[i].string);
- } else {
- match = !strcmp(url, url_table[i].string);
- }
- if (match) {
- result = url_table[i].handler(conn, &args);
- goto done;
- }
- }
-
- /* we didn't recognize the url */
- write_short_http_response(conn, 404, "Not found");
- result = 0;
-
- done:
- tor_free(url_mem);
- return result;
-}
-
-/** Helper function for GET / or GET /tor/
- */
-static int
-handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
-{
- (void) args; /* unused */
- const char *frontpage = get_dirportfrontpage();
-
- if (frontpage) {
- size_t dlen;
- dlen = strlen(frontpage);
- /* Let's return a disclaimer page (users shouldn't use V1 anymore,
- and caches don't fetch '/', so this is safe). */
-
- /* [We don't check for write_bucket_low here, since we want to serve
- * this page no matter what.] */
- write_http_response_header_impl(conn, dlen, "text/html", "identity",
- NULL, DIRPORTFRONTPAGE_CACHE_LIFETIME);
- connection_buf_add(frontpage, dlen, TO_CONN(conn));
- } else {
- write_short_http_response(conn, 404, "Not found");
- }
- return 0;
-}
-
-/** Warn that the cached consensus <b>consensus</b> of type
- * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
- * warning to avoid logging an entry on every request.
- */
-static void
-warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
- const char *flavor, time_t now)
-{
-#define TOO_OLD_WARNING_INTERVAL (60*60)
- static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
- char timestamp[ISO_TIME_LEN+1];
- time_t valid_until;
- char *dupes;
-
- if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
- return;
-
- if ((dupes = rate_limit_log(&warned, now))) {
- format_local_iso_time(timestamp, valid_until);
- log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
- "serve it to clients. It was valid until %s local time and we "
- "continued to serve it for up to 24 hours after it expired.%s",
- flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
- tor_free(dupes);
- }
-}
-
-/**
- * Parse a single hex-encoded sha3-256 digest from <b>hex</b> into
- * <b>digest</b>. Return 0 on success. On failure, report that the hash came
- * from <b>location</b>, report that we are taking <b>action</b> with it, and
- * return -1.
- */
-static int
-parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
- const char *action)
-{
- if (base16_decode((char*)digest, DIGEST256_LEN, hex, strlen(hex)) ==
- DIGEST256_LEN) {
- return 0;
- } else {
- log_fn(LOG_PROTOCOL_WARN, LD_DIR,
- "%s contained bogus digest %s; %s.",
- location, escaped(hex), action);
- return -1;
- }
-}
-
-/** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>,
- * set <b>digest_out<b> to a new smartlist containing every 256-bit
- * hex-encoded digest listed in that header and return 0. Otherwise return
- * -1. */
-static int
-parse_or_diff_from_header(smartlist_t **digests_out, const char *headers)
-{
- char *hdr = http_get_header(headers, X_OR_DIFF_FROM_CONSENSUS_HEADER);
- if (hdr == NULL) {
- return -1;
- }
- smartlist_t *hex_digests = smartlist_new();
- *digests_out = smartlist_new();
- smartlist_split_string(hex_digests, hdr, " ",
- SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
- SMARTLIST_FOREACH_BEGIN(hex_digests, const char *, hex) {
- uint8_t digest[DIGEST256_LEN];
- if (!parse_one_diff_hash(digest, hex, "X-Or-Diff-From-Consensus header",
- "ignoring")) {
- smartlist_add(*digests_out, tor_memdup(digest, sizeof(digest)));
- }
- } SMARTLIST_FOREACH_END(hex);
- SMARTLIST_FOREACH(hex_digests, char *, cp, tor_free(cp));
- smartlist_free(hex_digests);
- tor_free(hdr);
- return 0;
-}
-
-/** Fallback compression method. The fallback compression method is used in
- * case a client requests a non-compressed document. We only store compressed
- * documents, so we use this compression method to fetch the document and let
- * the spooling system do the streaming decompression.
- */
-#define FALLBACK_COMPRESS_METHOD ZLIB_METHOD
-
-/**
- * Try to find the best consensus diff possible in order to serve a client
- * request for a diff from one of the consensuses in <b>digests</b> to the
- * current consensus of flavor <b>flav</b>. The client supports the
- * compression methods listed in the <b>compression_methods</b> bitfield:
- * place the method chosen (if any) into <b>compression_used_out</b>.
- */
-static struct consensus_cache_entry_t *
-find_best_diff(const smartlist_t *digests, int flav,
- unsigned compression_methods,
- compress_method_t *compression_used_out)
-{
- struct consensus_cache_entry_t *result = NULL;
-
- SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
- unsigned u;
- for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
- compress_method_t method = srv_meth_pref_precompressed[u];
- if (0 == (compression_methods & (1u<<method)))
- continue; // client doesn't like this one, or we don't have it.
- if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256,
- diff_from, DIGEST256_LEN,
- method) == CONSDIFF_AVAILABLE) {
- tor_assert_nonfatal(result);
- *compression_used_out = method;
- return result;
- }
- }
- } SMARTLIST_FOREACH_END(diff_from);
-
- SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, diff_from) {
- if (consdiffmgr_find_diff_from(&result, flav, DIGEST_SHA3_256, diff_from,
- DIGEST256_LEN, FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
- tor_assert_nonfatal(result);
- *compression_used_out = FALLBACK_COMPRESS_METHOD;
- return result;
- }
- } SMARTLIST_FOREACH_END(diff_from);
-
- return NULL;
-}
-
-/** Lookup the cached consensus document by the flavor found in <b>flav</b>.
- * The preferred set of compression methods should be listed in the
- * <b>compression_methods</b> bitfield. The compression method chosen (if any)
- * is stored in <b>compression_used_out</b>. */
-static struct consensus_cache_entry_t *
-find_best_consensus(int flav,
- unsigned compression_methods,
- compress_method_t *compression_used_out)
-{
- struct consensus_cache_entry_t *result = NULL;
- unsigned u;
-
- for (u = 0; u < ARRAY_LENGTH(srv_meth_pref_precompressed); ++u) {
- compress_method_t method = srv_meth_pref_precompressed[u];
-
- if (0 == (compression_methods & (1u<<method)))
- continue;
-
- if (consdiffmgr_find_consensus(&result, flav,
- method) == CONSDIFF_AVAILABLE) {
- tor_assert_nonfatal(result);
- *compression_used_out = method;
- return result;
- }
- }
-
- if (consdiffmgr_find_consensus(&result, flav,
- FALLBACK_COMPRESS_METHOD) == CONSDIFF_AVAILABLE) {
- tor_assert_nonfatal(result);
- *compression_used_out = FALLBACK_COMPRESS_METHOD;
- return result;
- }
-
- return NULL;
-}
-
-/** Try to find the best supported compression method possible from a given
- * <b>compression_methods</b>. Return NO_METHOD if no mutually supported
- * compression method could be found. */
-static compress_method_t
-find_best_compression_method(unsigned compression_methods, int stream)
-{
- unsigned u;
- compress_method_t *methods;
- size_t length;
-
- if (stream) {
- methods = srv_meth_pref_streaming_compression;
- length = ARRAY_LENGTH(srv_meth_pref_streaming_compression);
- } else {
- methods = srv_meth_pref_precompressed;
- length = ARRAY_LENGTH(srv_meth_pref_precompressed);
- }
-
- for (u = 0; u < length; ++u) {
- compress_method_t method = methods[u];
- if (compression_methods & (1u<<method))
- return method;
- }
-
- return NO_METHOD;
-}
-
-/** Check if any of the digests in <b>digests</b> matches the latest consensus
- * flavor (given in <b>flavor</b>) that we have available. */
-static int
-digest_list_contains_best_consensus(consensus_flavor_t flavor,
- const smartlist_t *digests)
-{
- const networkstatus_t *ns = NULL;
-
- if (digests == NULL)
- return 0;
-
- ns = networkstatus_get_latest_consensus_by_flavor(flavor);
-
- if (ns == NULL)
- return 0;
-
- SMARTLIST_FOREACH_BEGIN(digests, const uint8_t *, digest) {
- if (tor_memeq(ns->digest_sha3_as_signed, digest, DIGEST256_LEN))
- return 1;
- } SMARTLIST_FOREACH_END(digest);
-
- return 0;
-}
-
-/** Check if the given compression method is allowed for a connection that is
- * supposed to be anonymous. Returns 1 if the compression method is allowed,
- * otherwise 0. */
-STATIC int
-allowed_anonymous_connection_compression_method(compress_method_t method)
-{
- unsigned u;
-
- for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression);
- ++u) {
- compress_method_t allowed_method =
- client_meth_allowed_anonymous_compression[u];
-
- if (! tor_compress_supports_method(allowed_method))
- continue;
-
- if (method == allowed_method)
- return 1;
- }
-
- return 0;
-}
-
-/** Log a warning when a remote server has sent us a document using a
- * compression method that is not allowed for anonymous directory requests. */
-STATIC void
-warn_disallowed_anonymous_compression_method(compress_method_t method)
-{
- log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
- "Received a %s HTTP response, which is not "
- "allowed for anonymous directory requests.",
- compression_method_get_human_name(method));
-}
-
-/** Encodes the results of parsing a consensus request to figure out what
- * consensus, and possibly what diffs, the user asked for. */
-typedef struct {
- /** name of the flavor to retrieve. */
- char *flavor;
- /** flavor to retrive, as enum. */
- consensus_flavor_t flav;
- /** plus-separated list of authority fingerprints; see
- * client_likes_consensus(). Aliases the URL in the request passed to
- * parse_consensus_request(). */
- const char *want_fps;
- /** Optionally, a smartlist of sha3 digests-as-signed of the consensuses
- * to return a diff from. */
- smartlist_t *diff_from_digests;
- /** If true, never send a full consensus. If there is no diff, send
- * a 404 instead. */
- int diff_only;
-} parsed_consensus_request_t;
-
-/** Remove all data held in <b>req</b>. Do not free <b>req</b> itself, since
- * it is stack-allocated. */
-static void
-parsed_consensus_request_clear(parsed_consensus_request_t *req)
-{
- if (!req)
- return;
- tor_free(req->flavor);
- if (req->diff_from_digests) {
- SMARTLIST_FOREACH(req->diff_from_digests, uint8_t *, d, tor_free(d));
- smartlist_free(req->diff_from_digests);
- }
- memset(req, 0, sizeof(parsed_consensus_request_t));
-}
-
-/**
- * Parse the URL and relevant headers of <b>args</b> for a current-consensus
- * request to learn what flavor of consensus we want, what keys it must be
- * signed with, and what diffs we would accept (or demand) instead. Return 0
- * on success and -1 on failure.
- */
-static int
-parse_consensus_request(parsed_consensus_request_t *out,
- const get_handler_args_t *args)
-{
- const char *url = args->url;
- memset(out, 0, sizeof(parsed_consensus_request_t));
- out->flav = FLAV_NS;
-
- const char CONSENSUS_URL_PREFIX[] = "/tor/status-vote/current/consensus/";
- const char CONSENSUS_FLAVORED_PREFIX[] =
- "/tor/status-vote/current/consensus-";
-
- /* figure out the flavor if any, and who we wanted to sign the thing */
- const char *after_flavor = NULL;
-
- if (!strcmpstart(url, CONSENSUS_FLAVORED_PREFIX)) {
- const char *f, *cp;
- f = url + strlen(CONSENSUS_FLAVORED_PREFIX);
- cp = strchr(f, '/');
- if (cp) {
- after_flavor = cp+1;
- out->flavor = tor_strndup(f, cp-f);
- } else {
- out->flavor = tor_strdup(f);
- }
- int flav = networkstatus_parse_flavor_name(out->flavor);
- if (flav < 0)
- flav = FLAV_NS;
- out->flav = flav;
- } else {
- if (!strcmpstart(url, CONSENSUS_URL_PREFIX))
- after_flavor = url+strlen(CONSENSUS_URL_PREFIX);
- }
-
- /* see whether we've been asked explicitly for a diff from an older
- * consensus. (The user might also have said that a diff would be okay,
- * via X-Or-Diff-From-Consensus */
- const char DIFF_COMPONENT[] = "diff/";
- char *diff_hash_in_url = NULL;
- if (after_flavor && !strcmpstart(after_flavor, DIFF_COMPONENT)) {
- after_flavor += strlen(DIFF_COMPONENT);
- const char *cp = strchr(after_flavor, '/');
- if (cp) {
- diff_hash_in_url = tor_strndup(after_flavor, cp-after_flavor);
- out->want_fps = cp+1;
- } else {
- diff_hash_in_url = tor_strdup(after_flavor);
- out->want_fps = NULL;
- }
- } else {
- out->want_fps = after_flavor;
- }
-
- if (diff_hash_in_url) {
- uint8_t diff_from[DIGEST256_LEN];
- out->diff_from_digests = smartlist_new();
- out->diff_only = 1;
- int ok = !parse_one_diff_hash(diff_from, diff_hash_in_url, "URL",
- "rejecting");
- tor_free(diff_hash_in_url);
- if (ok) {
- smartlist_add(out->diff_from_digests,
- tor_memdup(diff_from, DIGEST256_LEN));
- } else {
- return -1;
- }
- } else {
- parse_or_diff_from_header(&out->diff_from_digests, args->headers);
- }
-
- return 0;
-}
-
-/** Helper function for GET /tor/status-vote/current/consensus
- */
-static int
-handle_get_current_consensus(dir_connection_t *conn,
- const get_handler_args_t *args)
-{
- const compress_method_t compress_method =
- find_best_compression_method(args->compression_supported, 0);
- const time_t if_modified_since = args->if_modified_since;
- int clear_spool = 0;
-
- /* v3 network status fetch. */
- long lifetime = NETWORKSTATUS_CACHE_LIFETIME;
-
- time_t now = time(NULL);
- parsed_consensus_request_t req;
-
- if (parse_consensus_request(&req, args) < 0) {
- write_short_http_response(conn, 404, "Couldn't parse request");
- goto done;
- }
-
- if (digest_list_contains_best_consensus(req.flav,
- req.diff_from_digests)) {
- write_short_http_response(conn, 304, "Not modified");
- geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
- goto done;
- }
-
- struct consensus_cache_entry_t *cached_consensus = NULL;
-
- compress_method_t compression_used = NO_METHOD;
- if (req.diff_from_digests) {
- cached_consensus = find_best_diff(req.diff_from_digests, req.flav,
- args->compression_supported,
- &compression_used);
- }
-
- if (req.diff_only && !cached_consensus) {
- write_short_http_response(conn, 404, "No such diff available");
- // XXXX warn_consensus_is_too_old(v, req.flavor, now);
- geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
- goto done;
- }
-
- if (! cached_consensus) {
- cached_consensus = find_best_consensus(req.flav,
- args->compression_supported,
- &compression_used);
- }
-
- time_t fresh_until, valid_until;
- int have_fresh_until = 0, have_valid_until = 0;
- if (cached_consensus) {
- have_fresh_until =
- !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
- have_valid_until =
- !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
- }
-
- if (cached_consensus && have_valid_until &&
- !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
- write_short_http_response(conn, 404, "Consensus is too old");
- warn_consensus_is_too_old(cached_consensus, req.flavor, now);
- geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
- goto done;
- }
-
- if (cached_consensus && req.want_fps &&
- !client_likes_consensus(cached_consensus, req.want_fps)) {
- write_short_http_response(conn, 404, "Consensus not signed by sufficient "
- "number of requested authorities");
- geoip_note_ns_response(GEOIP_REJECT_NOT_ENOUGH_SIGS);
- goto done;
- }
-
- conn->spool = smartlist_new();
- clear_spool = 1;
- {
- spooled_resource_t *spooled;
- if (cached_consensus) {
- spooled = spooled_resource_new_from_cache_entry(cached_consensus);
- smartlist_add(conn->spool, spooled);
- }
- }
-
- lifetime = (have_fresh_until && fresh_until > now) ? fresh_until - now : 0;
-
- size_t size_guess = 0;
- int n_expired = 0;
- dirserv_spool_remove_missing_and_guess_size(conn, if_modified_since,
- compress_method != NO_METHOD,
- &size_guess,
- &n_expired);
-
- if (!smartlist_len(conn->spool) && !n_expired) {
- write_short_http_response(conn, 404, "Not found");
- geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
- goto done;
- } else if (!smartlist_len(conn->spool)) {
- write_short_http_response(conn, 304, "Not modified");
- geoip_note_ns_response(GEOIP_REJECT_NOT_MODIFIED);
- goto done;
- }
-
- if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
- log_debug(LD_DIRSERV,
- "Client asked for network status lists, but we've been "
- "writing too many bytes lately. Sending 503 Dir busy.");
- write_short_http_response(conn, 503, "Directory busy, try again later");
- geoip_note_ns_response(GEOIP_REJECT_BUSY);
- goto done;
- }
-
- tor_addr_t addr;
- if (tor_addr_parse(&addr, (TO_CONN(conn))->address) >= 0) {
- geoip_note_client_seen(GEOIP_CLIENT_NETWORKSTATUS,
- &addr, NULL,
- time(NULL));
- geoip_note_ns_response(GEOIP_SUCCESS);
- /* Note that a request for a network status has started, so that we
- * can measure the download time later on. */
- if (conn->dirreq_id)
- geoip_start_dirreq(conn->dirreq_id, size_guess, DIRREQ_TUNNELED);
- else
- geoip_start_dirreq(TO_CONN(conn)->global_identifier, size_guess,
- DIRREQ_DIRECT);
- }
-
- /* Use this header to tell caches that the response depends on the
- * X-Or-Diff-From-Consensus header (or lack thereof). */
- const char vary_header[] = "Vary: X-Or-Diff-From-Consensus\r\n";
-
- clear_spool = 0;
-
- // The compress_method might have been NO_METHOD, but we store the data
- // compressed. Decompress them using `compression_used`. See fallback code in
- // find_best_consensus() and find_best_diff().
- write_http_response_headers(conn, -1,
- compress_method == NO_METHOD ?
- NO_METHOD : compression_used,
- vary_header,
- smartlist_len(conn->spool) == 1 ? lifetime : 0);
-
- if (compress_method == NO_METHOD && smartlist_len(conn->spool))
- conn->compress_state = tor_compress_new(0, compression_used,
- HIGH_COMPRESSION);
-
- /* Prime the connection with some data. */
- const int initial_flush_result = connection_dirserv_flushed_some(conn);
- tor_assert_nonfatal(initial_flush_result == 0);
- goto done;
-
- done:
- parsed_consensus_request_clear(&req);
- if (clear_spool) {
- dir_conn_clear_spool(conn);
- }
- return 0;
-}
-
-/** Helper function for GET /tor/status-vote/{current,next}/...
- */
-static int
-handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
-{
- const char *url = args->url;
- {
- ssize_t body_len = 0;
- ssize_t estimated_len = 0;
- int lifetime = 60; /* XXXX?? should actually use vote intervals. */
- /* This smartlist holds strings that we can compress on the fly. */
- smartlist_t *items = smartlist_new();
- /* This smartlist holds cached_dir_t objects that have a precompressed
- * deflated version. */
- smartlist_t *dir_items = smartlist_new();
- dirvote_dirreq_get_status_vote(url, items, dir_items);
- if (!smartlist_len(dir_items) && !smartlist_len(items)) {
- write_short_http_response(conn, 404, "Not found");
- goto vote_done;
- }
-
- /* We're sending items from at most one kind of source */
- tor_assert_nonfatal(smartlist_len(items) == 0 ||
- smartlist_len(dir_items) == 0);
-
- int streaming;
- unsigned mask;
- if (smartlist_len(items)) {
- /* We're taking strings and compressing them on the fly. */
- streaming = 1;
- mask = ~0u;
- } else {
- /* We're taking cached_dir_t objects. We only have them uncompressed
- * or deflated. */
- streaming = 0;
- mask = (1u<<NO_METHOD) | (1u<<ZLIB_METHOD);
- }
- const compress_method_t compress_method = find_best_compression_method(
- args->compression_supported&mask, streaming);
-
- SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
- body_len += compress_method != NO_METHOD ?
- d->dir_compressed_len : d->dir_len);
- estimated_len += body_len;
- SMARTLIST_FOREACH(items, const char *, item, {
- size_t ln = strlen(item);
- if (compress_method != NO_METHOD) {
- estimated_len += ln/2;
- } else {
- body_len += ln; estimated_len += ln;
- }
- });
-
- if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
- write_short_http_response(conn, 503, "Directory busy, try again later");
- goto vote_done;
- }
- write_http_response_header(conn, body_len ? body_len : -1,
- compress_method,
- lifetime);
-
- if (smartlist_len(items)) {
- if (compress_method != NO_METHOD) {
- conn->compress_state = tor_compress_new(1, compress_method,
- choose_compression_level(estimated_len));
- SMARTLIST_FOREACH(items, const char *, c,
- connection_buf_add_compress(c, strlen(c), conn, 0));
- connection_buf_add_compress("", 0, conn, 1);
- } else {
- SMARTLIST_FOREACH(items, const char *, c,
- connection_buf_add(c, strlen(c), TO_CONN(conn)));
- }
- } else {
- SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
- connection_buf_add(compress_method != NO_METHOD ?
- d->dir_compressed : d->dir,
- compress_method != NO_METHOD ?
- d->dir_compressed_len : d->dir_len,
- TO_CONN(conn)));
- }
- vote_done:
- smartlist_free(items);
- smartlist_free(dir_items);
- goto done;
- }
- done:
- return 0;
-}
-
-/** Helper function for GET /tor/micro/d/...
- */
-static int
-handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
-{
- const char *url = args->url;
- const compress_method_t compress_method =
- find_best_compression_method(args->compression_supported, 1);
- int clear_spool = 1;
- {
- conn->spool = smartlist_new();
-
- dir_split_resource_into_spoolable(url+strlen("/tor/micro/d/"),
- DIR_SPOOL_MICRODESC,
- conn->spool, NULL,
- DSR_DIGEST256|DSR_BASE64|DSR_SORT_UNIQ);
-
- size_t size_guess = 0;
- dirserv_spool_remove_missing_and_guess_size(conn, 0,
- compress_method != NO_METHOD,
- &size_guess, NULL);
- if (smartlist_len(conn->spool) == 0) {
- write_short_http_response(conn, 404, "Not found");
- goto done;
- }
- if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
- log_info(LD_DIRSERV,
- "Client asked for server descriptors, but we've been "
- "writing too many bytes lately. Sending 503 Dir busy.");
- write_short_http_response(conn, 503, "Directory busy, try again later");
- goto done;
- }
-
- clear_spool = 0;
- write_http_response_header(conn, -1,
- compress_method,
- MICRODESC_CACHE_LIFETIME);
-
- if (compress_method != NO_METHOD)
- conn->compress_state = tor_compress_new(1, compress_method,
- choose_compression_level(size_guess));
-
- const int initial_flush_result = connection_dirserv_flushed_some(conn);
- tor_assert_nonfatal(initial_flush_result == 0);
- goto done;
- }
-
- done:
- if (clear_spool) {
- dir_conn_clear_spool(conn);
- }
- return 0;
-}
-
-/** Helper function for GET /tor/{server,extra}/...
- */
-static int
-handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
-{
- const char *url = args->url;
- const compress_method_t compress_method =
- find_best_compression_method(args->compression_supported, 1);
- const or_options_t *options = get_options();
- int clear_spool = 1;
- if (!strcmpstart(url,"/tor/server/") ||
- (!options->BridgeAuthoritativeDir &&
- !options->BridgeRelay && !strcmpstart(url,"/tor/extra/"))) {
- int res;
- const char *msg = NULL;
- int cache_lifetime = 0;
- int is_extra = !strcmpstart(url,"/tor/extra/");
- url += is_extra ? strlen("/tor/extra/") : strlen("/tor/server/");
- dir_spool_source_t source;
- time_t publish_cutoff = 0;
- if (!strcmpstart(url, "d/")) {
- source =
- is_extra ? DIR_SPOOL_EXTRA_BY_DIGEST : DIR_SPOOL_SERVER_BY_DIGEST;
- } else {
- source =
- is_extra ? DIR_SPOOL_EXTRA_BY_FP : DIR_SPOOL_SERVER_BY_FP;
- /* We only want to apply a publish cutoff when we're requesting
- * resources by fingerprint. */
- publish_cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
- }
-
- conn->spool = smartlist_new();
- res = dirserv_get_routerdesc_spool(conn->spool, url,
- source,
- connection_dir_is_encrypted(conn),
- &msg);
-
- if (!strcmpstart(url, "all")) {
- cache_lifetime = FULL_DIR_CACHE_LIFETIME;
- } else if (smartlist_len(conn->spool) == 1) {
- cache_lifetime = ROUTERDESC_BY_DIGEST_CACHE_LIFETIME;
- }
-
- size_t size_guess = 0;
- int n_expired = 0;
- dirserv_spool_remove_missing_and_guess_size(conn, publish_cutoff,
- compress_method != NO_METHOD,
- &size_guess, &n_expired);
-
- /* If we are the bridge authority and the descriptor is a bridge
- * descriptor, remember that we served this descriptor for desc stats. */
- /* XXXX it's a bit of a kludge to have this here. */
- if (get_options()->BridgeAuthoritativeDir &&
- source == DIR_SPOOL_SERVER_BY_FP) {
- SMARTLIST_FOREACH_BEGIN(conn->spool, spooled_resource_t *, spooled) {
- const routerinfo_t *router =
- router_get_by_id_digest((const char *)spooled->digest);
- /* router can be NULL here when the bridge auth is asked for its own
- * descriptor. */
- if (router && router->purpose == ROUTER_PURPOSE_BRIDGE)
- rep_hist_note_desc_served(router->cache_info.identity_digest);
- } SMARTLIST_FOREACH_END(spooled);
- }
-
- if (res < 0 || size_guess == 0 || smartlist_len(conn->spool) == 0) {
- if (msg == NULL)
- msg = "Not found";
- write_short_http_response(conn, 404, msg);
- } else {
- if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
- log_info(LD_DIRSERV,
- "Client asked for server descriptors, but we've been "
- "writing too many bytes lately. Sending 503 Dir busy.");
- write_short_http_response(conn, 503,
- "Directory busy, try again later");
- dir_conn_clear_spool(conn);
- goto done;
- }
- write_http_response_header(conn, -1, compress_method, cache_lifetime);
- if (compress_method != NO_METHOD)
- conn->compress_state = tor_compress_new(1, compress_method,
- choose_compression_level(size_guess));
- clear_spool = 0;
- /* Prime the connection with some data. */
- int initial_flush_result = connection_dirserv_flushed_some(conn);
- tor_assert_nonfatal(initial_flush_result == 0);
- }
- goto done;
- }
- done:
- if (clear_spool)
- dir_conn_clear_spool(conn);
- return 0;
-}
-
-/** Helper function for GET /tor/keys/...
- */
-static int
-handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
-{
- const char *url = args->url;
- const compress_method_t compress_method =
- find_best_compression_method(args->compression_supported, 1);
- const time_t if_modified_since = args->if_modified_since;
- {
- smartlist_t *certs = smartlist_new();
- ssize_t len = -1;
- if (!strcmp(url, "/tor/keys/all")) {
- authority_cert_get_all(certs);
- } else if (!strcmp(url, "/tor/keys/authority")) {
- authority_cert_t *cert = get_my_v3_authority_cert();
- if (cert)
- smartlist_add(certs, cert);
- } else if (!strcmpstart(url, "/tor/keys/fp/")) {
- smartlist_t *fps = smartlist_new();
- dir_split_resource_into_fingerprints(url+strlen("/tor/keys/fp/"),
- fps, NULL,
- DSR_HEX|DSR_SORT_UNIQ);
- SMARTLIST_FOREACH(fps, char *, d, {
- authority_cert_t *c = authority_cert_get_newest_by_id(d);
- if (c) smartlist_add(certs, c);
- tor_free(d);
- });
- smartlist_free(fps);
- } else if (!strcmpstart(url, "/tor/keys/sk/")) {
- smartlist_t *fps = smartlist_new();
- dir_split_resource_into_fingerprints(url+strlen("/tor/keys/sk/"),
- fps, NULL,
- DSR_HEX|DSR_SORT_UNIQ);
- SMARTLIST_FOREACH(fps, char *, d, {
- authority_cert_t *c = authority_cert_get_by_sk_digest(d);
- if (c) smartlist_add(certs, c);
- tor_free(d);
- });
- smartlist_free(fps);
- } else if (!strcmpstart(url, "/tor/keys/fp-sk/")) {
- smartlist_t *fp_sks = smartlist_new();
- dir_split_resource_into_fingerprint_pairs(url+strlen("/tor/keys/fp-sk/"),
- fp_sks);
- SMARTLIST_FOREACH(fp_sks, fp_pair_t *, pair, {
- authority_cert_t *c = authority_cert_get_by_digests(pair->first,
- pair->second);
- if (c) smartlist_add(certs, c);
- tor_free(pair);
- });
- smartlist_free(fp_sks);
- } else {
- write_short_http_response(conn, 400, "Bad request");
- goto keys_done;
- }
- if (!smartlist_len(certs)) {
- write_short_http_response(conn, 404, "Not found");
- goto keys_done;
- }
- SMARTLIST_FOREACH(certs, authority_cert_t *, c,
- if (c->cache_info.published_on < if_modified_since)
- SMARTLIST_DEL_CURRENT(certs, c));
- if (!smartlist_len(certs)) {
- write_short_http_response(conn, 304, "Not modified");
- goto keys_done;
- }
- len = 0;
- SMARTLIST_FOREACH(certs, authority_cert_t *, c,
- len += c->cache_info.signed_descriptor_len);
-
- if (global_write_bucket_low(TO_CONN(conn),
- compress_method != NO_METHOD ? len/2 : len,
- 2)) {
- write_short_http_response(conn, 503, "Directory busy, try again later");
- goto keys_done;
- }
-
- write_http_response_header(conn,
- compress_method != NO_METHOD ? -1 : len,
- compress_method,
- 60*60);
- if (compress_method != NO_METHOD) {
- conn->compress_state = tor_compress_new(1, compress_method,
- choose_compression_level(len));
- SMARTLIST_FOREACH(certs, authority_cert_t *, c,
- connection_buf_add_compress(
- c->cache_info.signed_descriptor_body,
- c->cache_info.signed_descriptor_len,
- conn, 0));
- connection_buf_add_compress("", 0, conn, 1);
- } else {
- SMARTLIST_FOREACH(certs, authority_cert_t *, c,
- connection_buf_add(c->cache_info.signed_descriptor_body,
- c->cache_info.signed_descriptor_len,
- TO_CONN(conn)));
- }
- keys_done:
- smartlist_free(certs);
- goto done;
- }
- done:
- return 0;
-}
-
-/** Helper function for GET /tor/rendezvous2/
- */
-static int
-handle_get_hs_descriptor_v2(dir_connection_t *conn,
- const get_handler_args_t *args)
-{
- const char *url = args->url;
- if (connection_dir_is_encrypted(conn)) {
- /* Handle v2 rendezvous descriptor fetch request. */
- const char *descp;
- const char *query = url + strlen("/tor/rendezvous2/");
- if (rend_valid_descriptor_id(query)) {
- log_info(LD_REND, "Got a v2 rendezvous descriptor request for ID '%s'",
- safe_str(escaped(query)));
- switch (rend_cache_lookup_v2_desc_as_dir(query, &descp)) {
- case 1: /* valid */
- write_http_response_header(conn, strlen(descp), NO_METHOD, 0);
- connection_buf_add(descp, strlen(descp), TO_CONN(conn));
- break;
- case 0: /* well-formed but not present */
- write_short_http_response(conn, 404, "Not found");
- break;
- case -1: /* not well-formed */
- write_short_http_response(conn, 400, "Bad request");
- break;
- }
- } else { /* not well-formed */
- write_short_http_response(conn, 400, "Bad request");
- }
- goto done;
- } else {
- /* Not encrypted! */
- write_short_http_response(conn, 404, "Not found");
- }
- done:
- return 0;
-}
-
-/** Helper function for GET /tor/hs/3/<z>. Only for version 3.
- */
-STATIC int
-handle_get_hs_descriptor_v3(dir_connection_t *conn,
- const get_handler_args_t *args)
-{
- int retval;
- const char *desc_str = NULL;
- const char *pubkey_str = NULL;
- const char *url = args->url;
-
- /* Reject unencrypted dir connections */
- if (!connection_dir_is_encrypted(conn)) {
- write_short_http_response(conn, 404, "Not found");
- goto done;
- }
-
- /* After the path prefix follows the base64 encoded blinded pubkey which we
- * use to get the descriptor from the cache. Skip the prefix and get the
- * pubkey. */
- tor_assert(!strcmpstart(url, "/tor/hs/3/"));
- pubkey_str = url + strlen("/tor/hs/3/");
- retval = hs_cache_lookup_as_dir(HS_VERSION_THREE,
- pubkey_str, &desc_str);
- if (retval <= 0 || desc_str == NULL) {
- write_short_http_response(conn, 404, "Not found");
- goto done;
- }
-
- /* Found requested descriptor! Pass it to this nice client. */
- write_http_response_header(conn, strlen(desc_str), NO_METHOD, 0);
- connection_buf_add(desc_str, strlen(desc_str), TO_CONN(conn));
-
- done:
- return 0;
-}
-
-/** Helper function for GET /tor/networkstatus-bridges
- */
-static int
-handle_get_networkstatus_bridges(dir_connection_t *conn,
- const get_handler_args_t *args)
-{
- const char *headers = args->headers;
-
- const or_options_t *options = get_options();
- if (options->BridgeAuthoritativeDir &&
- options->BridgePassword_AuthDigest_ &&
- connection_dir_is_encrypted(conn)) {
- char *status;
- char digest[DIGEST256_LEN];
-
- char *header = http_get_header(headers, "Authorization: Basic ");
- if (header)
- crypto_digest256(digest, header, strlen(header), DIGEST_SHA256);
-
- /* now make sure the password is there and right */
- if (!header ||
- tor_memneq(digest,
- options->BridgePassword_AuthDigest_, DIGEST256_LEN)) {
- write_short_http_response(conn, 404, "Not found");
- tor_free(header);
- goto done;
- }
- tor_free(header);
-
- /* all happy now. send an answer. */
- status = networkstatus_getinfo_by_purpose("bridge", time(NULL));
- size_t dlen = strlen(status);
- write_http_response_header(conn, dlen, NO_METHOD, 0);
- connection_buf_add(status, dlen, TO_CONN(conn));
- tor_free(status);
- goto done;
- }
- done:
- return 0;
-}
-
-/** Helper function for GET robots.txt or /tor/robots.txt */
-static int
-handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
-{
- (void)args;
- {
- const char robots[] = "User-agent: *\r\nDisallow: /\r\n";
- size_t len = strlen(robots);
- write_http_response_header(conn, len, NO_METHOD, ROBOTS_CACHE_LIFETIME);
- connection_buf_add(robots, len, TO_CONN(conn));
- }
- return 0;
-}
-
-/* Given the <b>url</b> from a POST request, try to extract the version number
- * using the provided <b>prefix</b>. The version should be after the prefix and
- * ending with the separator "/". For instance:
- * /tor/hs/3/publish
- *
- * On success, <b>end_pos</b> points to the position right after the version
- * was found. On error, it is set to NULL.
- *
- * Return version on success else negative value. */
-STATIC int
-parse_hs_version_from_post(const char *url, const char *prefix,
- const char **end_pos)
-{
- int ok;
- unsigned long version;
- const char *start;
- char *end = NULL;
-
- tor_assert(url);
- tor_assert(prefix);
- tor_assert(end_pos);
-
- /* Check if the prefix does start the url. */
- if (strcmpstart(url, prefix)) {
- goto err;
- }
- /* Move pointer to the end of the prefix string. */
- start = url + strlen(prefix);
- /* Try this to be the HS version and if we are still at the separator, next
- * will be move to the right value. */
- version = tor_parse_long(start, 10, 0, INT_MAX, &ok, &end);
- if (!ok) {
- goto err;
- }
-
- *end_pos = end;
- return (int) version;
- err:
- *end_pos = NULL;
- return -1;
-}
-
-/* Handle the POST request for a hidden service descripror. The request is in
- * <b>url</b>, the body of the request is in <b>body</b>. Return 200 on success
- * else return 400 indicating a bad request. */
-STATIC int
-handle_post_hs_descriptor(const char *url, const char *body)
-{
- int version;
- const char *end_pos;
-
- tor_assert(url);
- tor_assert(body);
-
- version = parse_hs_version_from_post(url, "/tor/hs/", &end_pos);
- if (version < 0) {
- goto err;
- }
-
- /* We have a valid version number, now make sure it's a publish request. Use
- * the end position just after the version and check for the command. */
- if (strcmpstart(end_pos, "/publish")) {
- goto err;
- }
-
- switch (version) {
- case HS_VERSION_THREE:
- if (hs_cache_store_as_dir(body) < 0) {
- goto err;
- }
- log_info(LD_REND, "Publish request for HS descriptor handled "
- "successfully.");
- break;
- default:
- /* Unsupported version, return a bad request. */
- goto err;
- }
-
- return 200;
- err:
- /* Bad request. */
- return 400;
-}
-
-/** Helper function: called when a dirserver gets a complete HTTP POST
- * request. Look for an uploaded server descriptor or rendezvous
- * service descriptor. On finding one, process it and write a
- * response into conn-\>outbuf. If the request is unrecognized, send a
- * 400. Always return 0. */
-MOCK_IMPL(STATIC int,
-directory_handle_command_post,(dir_connection_t *conn, const char *headers,
- const char *body, size_t body_len))
-{
- char *url = NULL;
- const or_options_t *options = get_options();
-
- log_debug(LD_DIRSERV,"Received POST command.");
-
- conn->base_.state = DIR_CONN_STATE_SERVER_WRITING;
-
- if (!public_server_mode(options)) {
- log_info(LD_DIR, "Rejected dir post request from %s "
- "since we're not a public relay.", conn->base_.address);
- write_short_http_response(conn, 503, "Not acting as a public relay");
- goto done;
- }
-
- if (parse_http_url(headers, &url) < 0) {
- write_short_http_response(conn, 400, "Bad request");
- return 0;
- }
- log_debug(LD_DIRSERV,"rewritten url as '%s'.", escaped(url));
-
- /* Handle v2 rendezvous service publish request. */
- if (connection_dir_is_encrypted(conn) &&
- !strcmpstart(url,"/tor/rendezvous2/publish")) {
- if (rend_cache_store_v2_desc_as_dir(body) < 0) {
- log_warn(LD_REND, "Rejected v2 rend descriptor (body size %d) from %s.",
- (int)body_len, conn->base_.address);
- write_short_http_response(conn, 400,
- "Invalid v2 service descriptor rejected");
- } else {
- write_short_http_response(conn, 200, "Service descriptor (v2) stored");
- log_info(LD_REND, "Handled v2 rendezvous descriptor post: accepted");
- }
- goto done;
- }
-
- /* Handle HS descriptor publish request. */
- /* XXX: This should be disabled with a consensus param until we want to
- * the prop224 be deployed and thus use. */
- if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
- const char *msg = "HS descriptor stored successfully.";
-
- /* We most probably have a publish request for an HS descriptor. */
- int code = handle_post_hs_descriptor(url, body);
- if (code != 200) {
- msg = "Invalid HS descriptor. Rejected.";
- }
- write_short_http_response(conn, code, msg);
- goto done;
- }
-
- if (!authdir_mode(options)) {
- /* we just provide cached directories; we don't want to
- * receive anything. */
- write_short_http_response(conn, 400, "Nonauthoritative directory does not "
- "accept posted server descriptors");
- goto done;
- }
+ unsigned u;
- if (authdir_mode(options) &&
- !strcmp(url,"/tor/")) { /* server descriptor post */
- const char *msg = "[None]";
- uint8_t purpose = authdir_mode_bridge(options) ?
- ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
- was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose,
- conn->base_.address, &msg);
- tor_assert(msg);
-
- if (r == ROUTER_ADDED_SUCCESSFULLY) {
- write_short_http_response(conn, 200, msg);
- } else if (WRA_WAS_OUTDATED(r)) {
- write_http_response_header_impl(conn, -1, NULL, NULL,
- "X-Descriptor-Not-New: Yes\r\n", -1);
- } else {
- log_info(LD_DIRSERV,
- "Rejected router descriptor or extra-info from %s "
- "(\"%s\").",
- conn->base_.address, msg);
- write_short_http_response(conn, 400, msg);
- }
- goto done;
- }
+ for (u = 0; u < ARRAY_LENGTH(client_meth_allowed_anonymous_compression);
+ ++u) {
+ compress_method_t allowed_method =
+ client_meth_allowed_anonymous_compression[u];
- if (authdir_mode_v3(options) &&
- !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */
- const char *msg = "OK";
- int status;
- if (dirvote_add_vote(body, &msg, &status)) {
- write_short_http_response(conn, status, "Vote stored");
- } else {
- tor_assert(msg);
- log_warn(LD_DIRSERV, "Rejected vote from %s (\"%s\").",
- conn->base_.address, msg);
- write_short_http_response(conn, status, msg);
- }
- goto done;
- }
+ if (! tor_compress_supports_method(allowed_method))
+ continue;
- if (authdir_mode_v3(options) &&
- !strcmp(url,"/tor/post/consensus-signature")) { /* sigs on consensus. */
- const char *msg = NULL;
- if (dirvote_add_signatures(body, conn->base_.address, &msg)>=0) {
- write_short_http_response(conn, 200, msg?msg:"Signatures stored");
- } else {
- log_warn(LD_DIR, "Unable to store signatures posted by %s: %s",
- conn->base_.address, msg?msg:"???");
- write_short_http_response(conn, 400,
- msg?msg:"Unable to store signatures");
- }
- goto done;
+ if (method == allowed_method)
+ return 1;
}
- /* we didn't recognize the url */
- write_short_http_response(conn, 404, "Not found");
-
- done:
- tor_free(url);
return 0;
}
-/** Called when a dirserver receives data on a directory connection;
- * looks for an HTTP request. If the request is complete, remove it
- * from the inbuf, try to process it; otherwise, leave it on the
- * buffer. Return a 0 on success, or -1 on error.
- */
-STATIC int
-directory_handle_command(dir_connection_t *conn)
-{
- char *headers=NULL, *body=NULL;
- size_t body_len=0;
- int r;
-
- tor_assert(conn);
- tor_assert(conn->base_.type == CONN_TYPE_DIR);
-
- switch (connection_fetch_from_buf_http(TO_CONN(conn),
- &headers, MAX_HEADERS_SIZE,
- &body, &body_len, MAX_DIR_UL_SIZE, 0)) {
- case -1: /* overflow */
- log_warn(LD_DIRSERV,
- "Request too large from address '%s' to DirPort. Closing.",
- safe_str(conn->base_.address));
- return -1;
- case 0:
- log_debug(LD_DIRSERV,"command not all here yet.");
- return 0;
- /* case 1, fall through */
- }
-
- http_set_address_origin(headers, TO_CONN(conn));
- // we should escape headers here as well,
- // but we can't call escaped() twice, as it uses the same buffer
- //log_debug(LD_DIRSERV,"headers %s, body %s.", headers, escaped(body));
-
- if (!strncasecmp(headers,"GET",3))
- r = directory_handle_command_get(conn, headers, body, body_len);
- else if (!strncasecmp(headers,"POST",4))
- r = directory_handle_command_post(conn, headers, body, body_len);
- else {
- log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
- "Got headers %s with unknown command. Closing.",
- escaped(headers));
- r = -1;
- }
-
- tor_free(headers); tor_free(body);
- return r;
-}
-
-/** Write handler for directory connections; called when all data has
- * been flushed. Close the connection or wait for a response as
- * appropriate.
- */
-int
-connection_dir_finished_flushing(dir_connection_t *conn)
+/** Log a warning when a remote server has sent us a document using a
+ * compression method that is not allowed for anonymous directory requests. */
+STATIC void
+warn_disallowed_anonymous_compression_method(compress_method_t method)
{
- tor_assert(conn);
- tor_assert(conn->base_.type == CONN_TYPE_DIR);
-
- if (conn->base_.marked_for_close)
- return 0;
-
- /* Note that we have finished writing the directory response. For direct
- * connections this means we're done; for tunneled connections it's only
- * an intermediate step. */
- if (conn->dirreq_id)
- geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED,
- DIRREQ_FLUSHING_DIR_CONN_FINISHED);
- else
- geoip_change_dirreq_state(TO_CONN(conn)->global_identifier,
- DIRREQ_DIRECT,
- DIRREQ_FLUSHING_DIR_CONN_FINISHED);
- switch (conn->base_.state) {
- case DIR_CONN_STATE_CONNECTING:
- case DIR_CONN_STATE_CLIENT_SENDING:
- log_debug(LD_DIR,"client finished sending command.");
- conn->base_.state = DIR_CONN_STATE_CLIENT_READING;
- return 0;
- case DIR_CONN_STATE_SERVER_WRITING:
- if (conn->spool) {
- log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!");
- connection_mark_for_close(TO_CONN(conn));
- } else {
- log_debug(LD_DIRSERV, "Finished writing server response. Closing.");
- connection_mark_for_close(TO_CONN(conn));
- }
- return 0;
- default:
- log_warn(LD_BUG,"called in unexpected state %d.",
- conn->base_.state);
- tor_fragile_assert();
- return -1;
- }
- return 0;
+ log_fn(LOG_PROTOCOL_WARN, LD_HTTP,
+ "Received a %s HTTP response, which is not "
+ "allowed for anonymous directory requests.",
+ compression_method_get_human_name(method));
}
/* We just got a new consensus! If there are other in-progress requests
@@ -5278,430 +3119,6 @@ connection_dir_close_consensus_fetches(dir_connection_t *except_this_one,
} SMARTLIST_FOREACH_END(d);
smartlist_free(conns_to_close);
}
-
-/** Connected handler for directory connections: begin sending data to the
- * server, and return 0.
- * Only used when connections don't immediately connect. */
-int
-connection_dir_finished_connecting(dir_connection_t *conn)
-{
- tor_assert(conn);
- tor_assert(conn->base_.type == CONN_TYPE_DIR);
- tor_assert(conn->base_.state == DIR_CONN_STATE_CONNECTING);
-
- log_debug(LD_HTTP,"Dir connection to router %s:%u established.",
- conn->base_.address,conn->base_.port);
-
- /* start flushing conn */
- conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
- return 0;
-}
-
-/** Decide which download schedule we want to use based on descriptor type
- * in <b>dls</b> and <b>options</b>.
- *
- * Then, return the initial delay for that download schedule, in seconds.
- *
- * Helper function for download_status_increment_failure(),
- * download_status_reset(), and download_status_increment_attempt(). */
-STATIC int
-find_dl_min_delay(const download_status_t *dls, const or_options_t *options)
-{
- tor_assert(dls);
- tor_assert(options);
-
- switch (dls->schedule) {
- case DL_SCHED_GENERIC:
- /* Any other directory document */
- if (dir_server_mode(options)) {
- /* A directory authority or directory mirror */
- return options->TestingServerDownloadInitialDelay;
- } else {
- return options->TestingClientDownloadInitialDelay;
- }
- case DL_SCHED_CONSENSUS:
- if (!networkstatus_consensus_can_use_multiple_directories(options)) {
- /* A public relay */
- return options->TestingServerConsensusDownloadInitialDelay;
- } else {
- /* A client or bridge */
- if (networkstatus_consensus_is_bootstrapping(time(NULL))) {
- /* During bootstrapping */
- if (!networkstatus_consensus_can_use_extra_fallbacks(options)) {
- /* A bootstrapping client without extra fallback directories */
- return options->
- ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay;
- } else if (dls->want_authority) {
- /* A bootstrapping client with extra fallback directories, but
- * connecting to an authority */
- return
- options->ClientBootstrapConsensusAuthorityDownloadInitialDelay;
- } else {
- /* A bootstrapping client connecting to extra fallback directories
- */
- return
- options->ClientBootstrapConsensusFallbackDownloadInitialDelay;
- }
- } else {
- /* A client with a reasonably live consensus, with or without
- * certificates */
- return options->TestingClientConsensusDownloadInitialDelay;
- }
- }
- case DL_SCHED_BRIDGE:
- if (options->UseBridges && num_bridges_usable(0) > 0) {
- /* A bridge client that is sure that one or more of its bridges are
- * running can afford to wait longer to update bridge descriptors. */
- return options->TestingBridgeDownloadInitialDelay;
- } else {
- /* A bridge client which might have no running bridges, must try to
- * get bridge descriptors straight away. */
- return options->TestingBridgeBootstrapDownloadInitialDelay;
- }
- default:
- tor_assert(0);
- }
-
- /* Impossible, but gcc will fail with -Werror without a `return`. */
- return 0;
-}
-
-/** As next_random_exponential_delay() below, but does not compute a random
- * value. Instead, compute the range of values that
- * next_random_exponential_delay() should use when computing its random value.
- * Store the low bound into *<b>low_bound_out</b>, and the high bound into
- * *<b>high_bound_out</b>. Guarantees that the low bound is strictly less
- * than the high bound. */
-STATIC void
-next_random_exponential_delay_range(int *low_bound_out,
- int *high_bound_out,
- int delay,
- int base_delay)
-{
- // This is the "decorrelated jitter" approach, from
- // https://www.awsarchitectureblog.com/2015/03/backoff.html
- // The formula is
- // sleep = min(cap, random_between(base, sleep * 3))
-
- const int delay_times_3 = delay < INT_MAX/3 ? delay * 3 : INT_MAX;
- *low_bound_out = base_delay;
- if (delay_times_3 > base_delay) {
- *high_bound_out = delay_times_3;
- } else {
- *high_bound_out = base_delay+1;
- }
-}
-
-/** Advance one delay step. The algorithm will generate a random delay,
- * such that each failure is possibly (random) longer than the ones before.
- *
- * We then clamp that value to be no larger than max_delay, and return it.
- *
- * The <b>base_delay</b> parameter is lowest possible delay time (can't be
- * zero); the <b>backoff_position</b> parameter is the number of times we've
- * generated a delay; and the <b>delay</b> argument is the most recently used
- * delay.
- */
-STATIC int
-next_random_exponential_delay(int delay,
- int base_delay)
-{
- /* Check preconditions */
- if (BUG(delay < 0))
- delay = 0;
-
- if (base_delay < 1)
- base_delay = 1;
-
- int low_bound=0, high_bound=INT_MAX;
-
- next_random_exponential_delay_range(&low_bound, &high_bound,
- delay, base_delay);
-
- return crypto_rand_int_range(low_bound, high_bound);
-}
-
-/** Find the current delay for dls based on min_delay.
- *
- * This function sets dls->next_attempt_at based on now, and returns the delay.
- * Helper for download_status_increment_failure and
- * download_status_increment_attempt. */
-STATIC int
-download_status_schedule_get_delay(download_status_t *dls,
- int min_delay,
- time_t now)
-{
- tor_assert(dls);
- /* If we're using random exponential backoff, we do need min/max delay */
- tor_assert(min_delay >= 0);
-
- int delay = INT_MAX;
- uint8_t dls_schedule_position = (dls->increment_on
- == DL_SCHED_INCREMENT_ATTEMPT
- ? dls->n_download_attempts
- : dls->n_download_failures);
-
- /* Check if we missed a reset somehow */
- IF_BUG_ONCE(dls->last_backoff_position > dls_schedule_position) {
- dls->last_backoff_position = 0;
- dls->last_delay_used = 0;
- }
-
- if (dls_schedule_position > 0) {
- delay = dls->last_delay_used;
-
- while (dls->last_backoff_position < dls_schedule_position) {
- /* Do one increment step */
- delay = next_random_exponential_delay(delay, min_delay);
- /* Update our position */
- ++(dls->last_backoff_position);
- }
- } else {
- /* If we're just starting out, use the minimum delay */
- delay = min_delay;
- }
-
- /* Clamp it within min/max if we have them */
- if (min_delay >= 0 && delay < min_delay) delay = min_delay;
-
- /* Store it for next time */
- dls->last_backoff_position = dls_schedule_position;
- dls->last_delay_used = delay;
-
- /* A negative delay makes no sense. Knowing that delay is
- * non-negative allows us to safely do the wrapping check below. */
- tor_assert(delay >= 0);
-
- /* Avoid now+delay overflowing TIME_MAX, by comparing with a subtraction
- * that won't overflow (since delay is non-negative). */
- if (delay < INT_MAX && now <= TIME_MAX - delay) {
- dls->next_attempt_at = now+delay;
- } else {
- dls->next_attempt_at = TIME_MAX;
- }
-
- return delay;
-}
-
-/* Log a debug message about item, which increments on increment_action, has
- * incremented dls_n_download_increments times. The message varies based on
- * was_schedule_incremented (if not, not_incremented_response is logged), and
- * the values of increment, dls_next_attempt_at, and now.
- * Helper for download_status_increment_failure and
- * download_status_increment_attempt. */
-static void
-download_status_log_helper(const char *item, int was_schedule_incremented,
- const char *increment_action,
- const char *not_incremented_response,
- uint8_t dls_n_download_increments, int increment,
- time_t dls_next_attempt_at, time_t now)
-{
- if (item) {
- if (!was_schedule_incremented)
- log_debug(LD_DIR, "%s %s %d time(s); I'll try again %s.",
- item, increment_action, (int)dls_n_download_increments,
- not_incremented_response);
- else if (increment == 0)
- log_debug(LD_DIR, "%s %s %d time(s); I'll try again immediately.",
- item, increment_action, (int)dls_n_download_increments);
- else if (dls_next_attempt_at < TIME_MAX)
- log_debug(LD_DIR, "%s %s %d time(s); I'll try again in %d seconds.",
- item, increment_action, (int)dls_n_download_increments,
- (int)(dls_next_attempt_at-now));
- else
- log_debug(LD_DIR, "%s %s %d time(s); Giving up for a while.",
- item, increment_action, (int)dls_n_download_increments);
- }
-}
-
-/** Determine when a failed download attempt should be retried.
- * Called when an attempt to download <b>dls</b> has failed with HTTP status
- * <b>status_code</b>. Increment the failure count (if the code indicates a
- * real failure, or if we're a server) and set <b>dls</b>-\>next_attempt_at to
- * an appropriate time in the future and return it.
- * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the
- * failure count, and return a time in the far future for the next attempt (to
- * avoid an immediate retry). */
-time_t
-download_status_increment_failure(download_status_t *dls, int status_code,
- const char *item, int server, time_t now)
-{
- (void) status_code; // XXXX no longer used.
- (void) server; // XXXX no longer used.
- int increment = -1;
- int min_delay = 0;
-
- tor_assert(dls);
-
- /* dls wasn't reset before it was used */
- if (dls->next_attempt_at == 0) {
- download_status_reset(dls);
- }
-
- /* count the failure */
- if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) {
- ++dls->n_download_failures;
- }
-
- if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
- /* We don't find out that a failure-based schedule has attempted a
- * connection until that connection fails.
- * We'll never find out about successful connections, but this doesn't
- * matter, because schedules are reset after a successful download.
- */
- if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
- ++dls->n_download_attempts;
-
- /* only return a failure retry time if this schedule increments on failures
- */
- min_delay = find_dl_min_delay(dls, get_options());
- increment = download_status_schedule_get_delay(dls, min_delay, now);
- }
-
- download_status_log_helper(item, !dls->increment_on, "failed",
- "concurrently", dls->n_download_failures,
- increment,
- download_status_get_next_attempt_at(dls),
- now);
-
- if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) {
- /* stop this schedule retrying on failure, it will launch concurrent
- * connections instead */
- return TIME_MAX;
- } else {
- return download_status_get_next_attempt_at(dls);
- }
-}
-
-/** Determine when the next download attempt should be made when using an
- * attempt-based (potentially concurrent) download schedule.
- * Called when an attempt to download <b>dls</b> is being initiated.
- * Increment the attempt count and set <b>dls</b>-\>next_attempt_at to an
- * appropriate time in the future and return it.
- * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment
- * the attempts, and return a time in the far future (to avoid launching a
- * concurrent attempt). */
-time_t
-download_status_increment_attempt(download_status_t *dls, const char *item,
- time_t now)
-{
- int delay = -1;
- int min_delay = 0;
-
- tor_assert(dls);
-
- /* dls wasn't reset before it was used */
- if (dls->next_attempt_at == 0) {
- download_status_reset(dls);
- }
-
- if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
- /* this schedule should retry on failure, and not launch any concurrent
- attempts */
- log_warn(LD_BUG, "Tried to launch an attempt-based connection on a "
- "failure-based schedule.");
- return TIME_MAX;
- }
-
- if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
- ++dls->n_download_attempts;
-
- min_delay = find_dl_min_delay(dls, get_options());
- delay = download_status_schedule_get_delay(dls, min_delay, now);
-
- download_status_log_helper(item, dls->increment_on, "attempted",
- "on failure", dls->n_download_attempts,
- delay, download_status_get_next_attempt_at(dls),
- now);
-
- return download_status_get_next_attempt_at(dls);
-}
-
-static time_t
-download_status_get_initial_delay_from_now(const download_status_t *dls)
-{
- /* We use constant initial delays, even in exponential backoff
- * schedules. */
- return time(NULL) + find_dl_min_delay(dls, get_options());
-}
-
-/** Reset <b>dls</b> so that it will be considered downloadable
- * immediately, and/or to show that we don't need it anymore.
- *
- * Must be called to initialise a download schedule, otherwise the zeroth item
- * in the schedule will never be used.
- *
- * (We find the zeroth element of the download schedule, and set
- * next_attempt_at to be the appropriate offset from 'now'. In most
- * cases this means setting it to 'now', so the item will be immediately
- * downloadable; when using authorities with fallbacks, there is a few seconds'
- * delay.) */
-void
-download_status_reset(download_status_t *dls)
-{
- if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD
- || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD)
- return; /* Don't reset this. */
-
- dls->n_download_failures = 0;
- dls->n_download_attempts = 0;
- dls->next_attempt_at = download_status_get_initial_delay_from_now(dls);
- dls->last_backoff_position = 0;
- dls->last_delay_used = 0;
- /* Don't reset dls->want_authority or dls->increment_on */
-}
-
-/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is
- * ready to get its download reattempted. */
-int
-download_status_is_ready(download_status_t *dls, time_t now)
-{
- /* dls wasn't reset before it was used */
- if (dls->next_attempt_at == 0) {
- download_status_reset(dls);
- }
-
- return download_status_get_next_attempt_at(dls) <= now;
-}
-
-/** Mark <b>dl</b> as never downloadable. */
-void
-download_status_mark_impossible(download_status_t *dl)
-{
- dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD;
- dl->n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD;
-}
-
-/** Return the number of failures on <b>dls</b> since the last success (if
- * any). */
-int
-download_status_get_n_failures(const download_status_t *dls)
-{
- return dls->n_download_failures;
-}
-
-/** Return the number of attempts to download <b>dls</b> since the last success
- * (if any). This can differ from download_status_get_n_failures() due to
- * outstanding concurrent attempts. */
-int
-download_status_get_n_attempts(const download_status_t *dls)
-{
- return dls->n_download_attempts;
-}
-
-/** Return the next time to attempt to download <b>dls</b>. */
-time_t
-download_status_get_next_attempt_at(const download_status_t *dls)
-{
- /* dls wasn't reset before it was used */
- if (dls->next_attempt_at == 0) {
- /* so give the answer we would have given if it had been */
- return download_status_get_initial_delay_from_now(dls);
- }
-
- return dls->next_attempt_at;
-}
-
/** Called when one or more routerdesc (or extrainfo, if <b>was_extrainfo</b>)
* fetches have failed (with uppercase fingerprints listed in <b>failed</b>,
* either as descriptor digests or as identity digests based on
@@ -5784,189 +3201,3 @@ dir_microdesc_download_failed(smartlist_t *failed,
}
} SMARTLIST_FOREACH_END(d);
}
-
-/** Helper. Compare two fp_pair_t objects, and return negative, 0, or
- * positive as appropriate. */
-static int
-compare_pairs_(const void **a, const void **b)
-{
- const fp_pair_t *fp1 = *a, *fp2 = *b;
- int r;
- if ((r = fast_memcmp(fp1->first, fp2->first, DIGEST_LEN)))
- return r;
- else
- return fast_memcmp(fp1->second, fp2->second, DIGEST_LEN);
-}
-
-/** Divide a string <b>res</b> of the form FP1-FP2+FP3-FP4...[.z], where each
- * FP is a hex-encoded fingerprint, into a sequence of distinct sorted
- * fp_pair_t. Skip malformed pairs. On success, return 0 and add those
- * fp_pair_t into <b>pairs_out</b>. On failure, return -1. */
-int
-dir_split_resource_into_fingerprint_pairs(const char *res,
- smartlist_t *pairs_out)
-{
- smartlist_t *pairs_tmp = smartlist_new();
- smartlist_t *pairs_result = smartlist_new();
-
- smartlist_split_string(pairs_tmp, res, "+", 0, 0);
- if (smartlist_len(pairs_tmp)) {
- char *last = smartlist_get(pairs_tmp,smartlist_len(pairs_tmp)-1);
- size_t last_len = strlen(last);
- if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
- last[last_len-2] = '\0';
- }
- }
- SMARTLIST_FOREACH_BEGIN(pairs_tmp, char *, cp) {
- if (strlen(cp) != HEX_DIGEST_LEN*2+1) {
- log_info(LD_DIR,
- "Skipping digest pair %s with non-standard length.", escaped(cp));
- } else if (cp[HEX_DIGEST_LEN] != '-') {
- log_info(LD_DIR,
- "Skipping digest pair %s with missing dash.", escaped(cp));
- } else {
- fp_pair_t pair;
- if (base16_decode(pair.first, DIGEST_LEN,
- cp, HEX_DIGEST_LEN) != DIGEST_LEN ||
- base16_decode(pair.second,DIGEST_LEN,
- cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN) != DIGEST_LEN) {
- log_info(LD_DIR, "Skipping non-decodable digest pair %s", escaped(cp));
- } else {
- smartlist_add(pairs_result, tor_memdup(&pair, sizeof(pair)));
- }
- }
- tor_free(cp);
- } SMARTLIST_FOREACH_END(cp);
- smartlist_free(pairs_tmp);
-
- /* Uniq-and-sort */
- smartlist_sort(pairs_result, compare_pairs_);
- smartlist_uniq(pairs_result, compare_pairs_, tor_free_);
-
- smartlist_add_all(pairs_out, pairs_result);
- smartlist_free(pairs_result);
- return 0;
-}
-
-/** Given a directory <b>resource</b> request, containing zero
- * or more strings separated by plus signs, followed optionally by ".z", store
- * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is
- * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.
- *
- * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and
- * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as
- * a separator, delete all the elements that aren't base64-encoded digests,
- * and decode the rest. If (flags & DSR_DIGEST256), these digests should be
- * 256 bits long; else they should be 160.
- *
- * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates.
- */
-int
-dir_split_resource_into_fingerprints(const char *resource,
- smartlist_t *fp_out, int *compressed_out,
- int flags)
-{
- const int decode_hex = flags & DSR_HEX;
- const int decode_base64 = flags & DSR_BASE64;
- const int digests_are_256 = flags & DSR_DIGEST256;
- const int sort_uniq = flags & DSR_SORT_UNIQ;
-
- const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN;
- const int hex_digest_len = digests_are_256 ?
- HEX_DIGEST256_LEN : HEX_DIGEST_LEN;
- const int base64_digest_len = digests_are_256 ?
- BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN;
- smartlist_t *fp_tmp = smartlist_new();
-
- tor_assert(!(decode_hex && decode_base64));
- tor_assert(fp_out);
-
- smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0);
- if (compressed_out)
- *compressed_out = 0;
- if (smartlist_len(fp_tmp)) {
- char *last = smartlist_get(fp_tmp,smartlist_len(fp_tmp)-1);
- size_t last_len = strlen(last);
- if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
- last[last_len-2] = '\0';
- if (compressed_out)
- *compressed_out = 1;
- }
- }
- if (decode_hex || decode_base64) {
- const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len;
- int i;
- char *cp, *d = NULL;
- for (i = 0; i < smartlist_len(fp_tmp); ++i) {
- cp = smartlist_get(fp_tmp, i);
- if (strlen(cp) != encoded_len) {
- log_info(LD_DIR,
- "Skipping digest %s with non-standard length.", escaped(cp));
- smartlist_del_keeporder(fp_tmp, i--);
- goto again;
- }
- d = tor_malloc_zero(digest_len);
- if (decode_hex ?
- (base16_decode(d, digest_len, cp, hex_digest_len) != digest_len) :
- (base64_decode(d, digest_len, cp, base64_digest_len)
- != digest_len)) {
- log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp));
- smartlist_del_keeporder(fp_tmp, i--);
- goto again;
- }
- smartlist_set(fp_tmp, i, d);
- d = NULL;
- again:
- tor_free(cp);
- tor_free(d);
- }
- }
- if (sort_uniq) {
- if (decode_hex || decode_base64) {
- if (digests_are_256) {
- smartlist_sort_digests256(fp_tmp);
- smartlist_uniq_digests256(fp_tmp);
- } else {
- smartlist_sort_digests(fp_tmp);
- smartlist_uniq_digests(fp_tmp);
- }
- } else {
- smartlist_sort_strings(fp_tmp);
- smartlist_uniq_strings(fp_tmp);
- }
- }
- smartlist_add_all(fp_out, fp_tmp);
- smartlist_free(fp_tmp);
- return 0;
-}
-
-/** As dir_split_resource_into_fingerprints, but instead fills
- * <b>spool_out</b> with a list of spoolable_resource_t for the resource
- * identified through <b>source</b>. */
-int
-dir_split_resource_into_spoolable(const char *resource,
- dir_spool_source_t source,
- smartlist_t *spool_out,
- int *compressed_out,
- int flags)
-{
- smartlist_t *fingerprints = smartlist_new();
-
- tor_assert(flags & (DSR_HEX|DSR_BASE64));
- const size_t digest_len =
- (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN;
-
- int r = dir_split_resource_into_fingerprints(resource, fingerprints,
- compressed_out, flags);
- /* This is not a very efficient implementation XXXX */
- SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) {
- spooled_resource_t *spooled =
- spooled_resource_new(source, digest, digest_len);
- if (spooled)
- smartlist_add(spool_out, spooled);
- tor_free(digest);
- } SMARTLIST_FOREACH_END(digest);
-
- smartlist_free(fingerprints);
- return r;
-}
diff --git a/src/feature/dirclient/dirclient.h b/src/feature/dirclient/dirclient.h
new file mode 100644
index 000000000..6bbff2f84
--- /dev/null
+++ b/src/feature/dirclient/dirclient.h
@@ -0,0 +1,172 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dirclient.h
+ * \brief Header file for dirclient.c.
+ **/
+
+#ifndef TOR_DIRCLIENT_H
+#define TOR_DIRCLIENT_H
+
+#include "feature/hs/hs_ident.h"
+
+int directories_have_accepted_server_descriptor(void);
+void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
+ dirinfo_type_t type, const char *payload,
+ size_t payload_len, size_t extrainfo_len);
+MOCK_DECL(void, directory_get_from_dirserver, (
+ uint8_t dir_purpose,
+ uint8_t router_purpose,
+ const char *resource,
+ int pds_flags,
+ download_want_authority_t want_authority));
+void directory_get_from_all_authorities(uint8_t dir_purpose,
+ uint8_t router_purpose,
+ const char *resource);
+
+/** Enumeration of ways to connect to a directory server */
+typedef enum {
+ /** Default: connect over a one-hop Tor circuit. Relays fall back to direct
+ * DirPort connections, clients, onion services, and bridges do not */
+ DIRIND_ONEHOP=0,
+ /** Connect over a multi-hop anonymizing Tor circuit */
+ DIRIND_ANONYMOUS=1,
+ /** Connect to the DirPort directly */
+ DIRIND_DIRECT_CONN,
+ /** Connect over a multi-hop anonymizing Tor circuit to our dirport */
+ DIRIND_ANON_DIRPORT,
+} dir_indirection_t;
+
+int directory_must_use_begindir(const or_options_t *options);
+
+/**
+ * A directory_request_t describes the information about a directory request
+ * at the client side. It describes what we're going to ask for, which
+ * directory we're going to ask for it, how we're going to contact that
+ * directory, and (in some cases) what to do with it when we're done.
+ */
+typedef struct directory_request_t directory_request_t;
+directory_request_t *directory_request_new(uint8_t dir_purpose);
+void directory_request_free_(directory_request_t *req);
+#define directory_request_free(req) \
+ FREE_AND_NULL(directory_request_t, directory_request_free_, (req))
+void directory_request_set_or_addr_port(directory_request_t *req,
+ const tor_addr_port_t *p);
+void directory_request_set_dir_addr_port(directory_request_t *req,
+ const tor_addr_port_t *p);
+void directory_request_set_directory_id_digest(directory_request_t *req,
+ const char *digest);
+struct circuit_guard_state_t;
+void directory_request_set_guard_state(directory_request_t *req,
+ struct circuit_guard_state_t *state);
+void directory_request_set_router_purpose(directory_request_t *req,
+ uint8_t router_purpose);
+void directory_request_set_indirection(directory_request_t *req,
+ dir_indirection_t indirection);
+void directory_request_set_resource(directory_request_t *req,
+ const char *resource);
+void directory_request_set_payload(directory_request_t *req,
+ const char *payload,
+ size_t payload_len);
+void directory_request_set_if_modified_since(directory_request_t *req,
+ time_t if_modified_since);
+void directory_request_set_rend_query(directory_request_t *req,
+ const rend_data_t *query);
+void directory_request_upload_set_hs_ident(directory_request_t *req,
+ const hs_ident_dir_conn_t *ident);
+void directory_request_fetch_set_hs_ident(directory_request_t *req,
+ const hs_ident_dir_conn_t *ident);
+
+void directory_request_set_routerstatus(directory_request_t *req,
+ const routerstatus_t *rs);
+void directory_request_add_header(directory_request_t *req,
+ const char *key,
+ const char *val);
+MOCK_DECL(void, directory_initiate_request, (directory_request_t *request));
+
+int router_supports_extrainfo(const char *identity_digest, int is_authority);
+
+void connection_dir_client_request_failed(dir_connection_t *conn);
+void connection_dir_client_refetch_hsdesc_if_needed(
+ dir_connection_t *dir_conn);
+
+#ifdef DIRCLIENT_PRIVATE
+struct directory_request_t {
+ /**
+ * These fields specify which directory we're contacting. Routerstatus,
+ * if present, overrides the other fields.
+ *
+ * @{ */
+ tor_addr_port_t or_addr_port;
+ tor_addr_port_t dir_addr_port;
+ char digest[DIGEST_LEN];
+
+ const routerstatus_t *routerstatus;
+ /** @} */
+ /** One of DIR_PURPOSE_* other than DIR_PURPOSE_SERVER. Describes what
+ * kind of operation we'll be doing (upload/download), and of what kind
+ * of document. */
+ uint8_t dir_purpose;
+ /** One of ROUTER_PURPOSE_*; used for uploads and downloads of routerinfo
+ * and extrainfo docs. */
+ uint8_t router_purpose;
+ /** Enum: determines whether to anonymize, and whether to use dirport or
+ * orport. */
+ dir_indirection_t indirection;
+ /** Alias to the variable part of the URL for this request */
+ const char *resource;
+ /** Alias to the payload to upload (if any) */
+ const char *payload;
+ /** Number of bytes to upload from payload</b> */
+ size_t payload_len;
+ /** Value to send in an if-modified-since header, or 0 for none. */
+ time_t if_modified_since;
+ /** Hidden-service-specific information v2. */
+ const rend_data_t *rend_query;
+ /** Extra headers to append to the request */
+ struct config_line_t *additional_headers;
+ /** Hidden-service-specific information for v3+. */
+ const hs_ident_dir_conn_t *hs_ident;
+ /** Used internally to directory.c: gets informed when the attempt to
+ * connect to the directory succeeds or fails, if that attempt bears on the
+ * directory's usability as a directory guard. */
+ struct circuit_guard_state_t *guard_state;
+};
+
+/** A structure to hold arguments passed into each directory response
+ * handler */
+typedef struct response_handler_args_t {
+ int status_code;
+ const char *reason;
+ const char *body;
+ size_t body_len;
+ const char *headers;
+} response_handler_args_t;
+
+enum compress_method_t;
+STATIC int allowed_anonymous_connection_compression_method(
+ enum compress_method_t);
+STATIC void warn_disallowed_anonymous_compression_method(
+ enum compress_method_t);
+
+STATIC int should_use_directory_guards(const or_options_t *options);
+STATIC char *accept_encoding_header(void);
+STATIC const char *dir_conn_purpose_to_string(int purpose);
+
+STATIC int handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
+ const response_handler_args_t *args);
+STATIC int handle_response_fetch_microdesc(dir_connection_t *conn,
+ const response_handler_args_t *args);
+
+STATIC int handle_response_fetch_consensus(dir_connection_t *conn,
+ const response_handler_args_t *args);
+
+STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
+ const char *resource);
+#endif
+
+#endif /* !defined(TOR_DIRCLIENT_H) */
diff --git a/src/feature/dirclient/dlstatus.c b/src/feature/dirclient/dlstatus.c
new file mode 100644
index 000000000..017281e00
--- /dev/null
+++ b/src/feature/dirclient/dlstatus.c
@@ -0,0 +1,422 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DLSTATUS_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "feature/client/entrynodes.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/router.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "feature/dirclient/download_status_st.h"
+
+/** Decide which download schedule we want to use based on descriptor type
+ * in <b>dls</b> and <b>options</b>.
+ *
+ * Then, return the initial delay for that download schedule, in seconds.
+ *
+ * Helper function for download_status_increment_failure(),
+ * download_status_reset(), and download_status_increment_attempt(). */
+STATIC int
+find_dl_min_delay(const download_status_t *dls, const or_options_t *options)
+{
+ tor_assert(dls);
+ tor_assert(options);
+
+ switch (dls->schedule) {
+ case DL_SCHED_GENERIC:
+ /* Any other directory document */
+ if (dir_server_mode(options)) {
+ /* A directory authority or directory mirror */
+ return options->TestingServerDownloadInitialDelay;
+ } else {
+ return options->TestingClientDownloadInitialDelay;
+ }
+ case DL_SCHED_CONSENSUS:
+ if (!networkstatus_consensus_can_use_multiple_directories(options)) {
+ /* A public relay */
+ return options->TestingServerConsensusDownloadInitialDelay;
+ } else {
+ /* A client or bridge */
+ if (networkstatus_consensus_is_bootstrapping(time(NULL))) {
+ /* During bootstrapping */
+ if (!networkstatus_consensus_can_use_extra_fallbacks(options)) {
+ /* A bootstrapping client without extra fallback directories */
+ return options->
+ ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay;
+ } else if (dls->want_authority) {
+ /* A bootstrapping client with extra fallback directories, but
+ * connecting to an authority */
+ return
+ options->ClientBootstrapConsensusAuthorityDownloadInitialDelay;
+ } else {
+ /* A bootstrapping client connecting to extra fallback directories
+ */
+ return
+ options->ClientBootstrapConsensusFallbackDownloadInitialDelay;
+ }
+ } else {
+ /* A client with a reasonably live consensus, with or without
+ * certificates */
+ return options->TestingClientConsensusDownloadInitialDelay;
+ }
+ }
+ case DL_SCHED_BRIDGE:
+ if (options->UseBridges && num_bridges_usable(0) > 0) {
+ /* A bridge client that is sure that one or more of its bridges are
+ * running can afford to wait longer to update bridge descriptors. */
+ return options->TestingBridgeDownloadInitialDelay;
+ } else {
+ /* A bridge client which might have no running bridges, must try to
+ * get bridge descriptors straight away. */
+ return options->TestingBridgeBootstrapDownloadInitialDelay;
+ }
+ default:
+ tor_assert(0);
+ }
+
+ /* Impossible, but gcc will fail with -Werror without a `return`. */
+ return 0;
+}
+
+/** As next_random_exponential_delay() below, but does not compute a random
+ * value. Instead, compute the range of values that
+ * next_random_exponential_delay() should use when computing its random value.
+ * Store the low bound into *<b>low_bound_out</b>, and the high bound into
+ * *<b>high_bound_out</b>. Guarantees that the low bound is strictly less
+ * than the high bound. */
+STATIC void
+next_random_exponential_delay_range(int *low_bound_out,
+ int *high_bound_out,
+ int delay,
+ int base_delay)
+{
+ // This is the "decorrelated jitter" approach, from
+ // https://www.awsarchitectureblog.com/2015/03/backoff.html
+ // The formula is
+ // sleep = min(cap, random_between(base, sleep * 3))
+
+ const int delay_times_3 = delay < INT_MAX/3 ? delay * 3 : INT_MAX;
+ *low_bound_out = base_delay;
+ if (delay_times_3 > base_delay) {
+ *high_bound_out = delay_times_3;
+ } else {
+ *high_bound_out = base_delay+1;
+ }
+}
+
+/** Advance one delay step. The algorithm will generate a random delay,
+ * such that each failure is possibly (random) longer than the ones before.
+ *
+ * We then clamp that value to be no larger than max_delay, and return it.
+ *
+ * The <b>base_delay</b> parameter is lowest possible delay time (can't be
+ * zero); the <b>backoff_position</b> parameter is the number of times we've
+ * generated a delay; and the <b>delay</b> argument is the most recently used
+ * delay.
+ */
+STATIC int
+next_random_exponential_delay(int delay,
+ int base_delay)
+{
+ /* Check preconditions */
+ if (BUG(delay < 0))
+ delay = 0;
+
+ if (base_delay < 1)
+ base_delay = 1;
+
+ int low_bound=0, high_bound=INT_MAX;
+
+ next_random_exponential_delay_range(&low_bound, &high_bound,
+ delay, base_delay);
+
+ return crypto_rand_int_range(low_bound, high_bound);
+}
+
+/** Find the current delay for dls based on min_delay.
+ *
+ * This function sets dls->next_attempt_at based on now, and returns the delay.
+ * Helper for download_status_increment_failure and
+ * download_status_increment_attempt. */
+STATIC int
+download_status_schedule_get_delay(download_status_t *dls,
+ int min_delay,
+ time_t now)
+{
+ tor_assert(dls);
+ /* If we're using random exponential backoff, we do need min/max delay */
+ tor_assert(min_delay >= 0);
+
+ int delay = INT_MAX;
+ uint8_t dls_schedule_position = (dls->increment_on
+ == DL_SCHED_INCREMENT_ATTEMPT
+ ? dls->n_download_attempts
+ : dls->n_download_failures);
+
+ /* Check if we missed a reset somehow */
+ IF_BUG_ONCE(dls->last_backoff_position > dls_schedule_position) {
+ dls->last_backoff_position = 0;
+ dls->last_delay_used = 0;
+ }
+
+ if (dls_schedule_position > 0) {
+ delay = dls->last_delay_used;
+
+ while (dls->last_backoff_position < dls_schedule_position) {
+ /* Do one increment step */
+ delay = next_random_exponential_delay(delay, min_delay);
+ /* Update our position */
+ ++(dls->last_backoff_position);
+ }
+ } else {
+ /* If we're just starting out, use the minimum delay */
+ delay = min_delay;
+ }
+
+ /* Clamp it within min/max if we have them */
+ if (min_delay >= 0 && delay < min_delay) delay = min_delay;
+
+ /* Store it for next time */
+ dls->last_backoff_position = dls_schedule_position;
+ dls->last_delay_used = delay;
+
+ /* A negative delay makes no sense. Knowing that delay is
+ * non-negative allows us to safely do the wrapping check below. */
+ tor_assert(delay >= 0);
+
+ /* Avoid now+delay overflowing TIME_MAX, by comparing with a subtraction
+ * that won't overflow (since delay is non-negative). */
+ if (delay < INT_MAX && now <= TIME_MAX - delay) {
+ dls->next_attempt_at = now+delay;
+ } else {
+ dls->next_attempt_at = TIME_MAX;
+ }
+
+ return delay;
+}
+
+/* Log a debug message about item, which increments on increment_action, has
+ * incremented dls_n_download_increments times. The message varies based on
+ * was_schedule_incremented (if not, not_incremented_response is logged), and
+ * the values of increment, dls_next_attempt_at, and now.
+ * Helper for download_status_increment_failure and
+ * download_status_increment_attempt. */
+static void
+download_status_log_helper(const char *item, int was_schedule_incremented,
+ const char *increment_action,
+ const char *not_incremented_response,
+ uint8_t dls_n_download_increments, int increment,
+ time_t dls_next_attempt_at, time_t now)
+{
+ if (item) {
+ if (!was_schedule_incremented)
+ log_debug(LD_DIR, "%s %s %d time(s); I'll try again %s.",
+ item, increment_action, (int)dls_n_download_increments,
+ not_incremented_response);
+ else if (increment == 0)
+ log_debug(LD_DIR, "%s %s %d time(s); I'll try again immediately.",
+ item, increment_action, (int)dls_n_download_increments);
+ else if (dls_next_attempt_at < TIME_MAX)
+ log_debug(LD_DIR, "%s %s %d time(s); I'll try again in %d seconds.",
+ item, increment_action, (int)dls_n_download_increments,
+ (int)(dls_next_attempt_at-now));
+ else
+ log_debug(LD_DIR, "%s %s %d time(s); Giving up for a while.",
+ item, increment_action, (int)dls_n_download_increments);
+ }
+}
+
+/** Determine when a failed download attempt should be retried.
+ * Called when an attempt to download <b>dls</b> has failed with HTTP status
+ * <b>status_code</b>. Increment the failure count (if the code indicates a
+ * real failure, or if we're a server) and set <b>dls</b>-\>next_attempt_at to
+ * an appropriate time in the future and return it.
+ * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_ATTEMPT, increment the
+ * failure count, and return a time in the far future for the next attempt (to
+ * avoid an immediate retry). */
+time_t
+download_status_increment_failure(download_status_t *dls, int status_code,
+ const char *item, int server, time_t now)
+{
+ (void) status_code; // XXXX no longer used.
+ (void) server; // XXXX no longer used.
+ int increment = -1;
+ int min_delay = 0;
+
+ tor_assert(dls);
+
+ /* dls wasn't reset before it was used */
+ if (dls->next_attempt_at == 0) {
+ download_status_reset(dls);
+ }
+
+ /* count the failure */
+ if (dls->n_download_failures < IMPOSSIBLE_TO_DOWNLOAD-1) {
+ ++dls->n_download_failures;
+ }
+
+ if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
+ /* We don't find out that a failure-based schedule has attempted a
+ * connection until that connection fails.
+ * We'll never find out about successful connections, but this doesn't
+ * matter, because schedules are reset after a successful download.
+ */
+ if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
+ ++dls->n_download_attempts;
+
+ /* only return a failure retry time if this schedule increments on failures
+ */
+ min_delay = find_dl_min_delay(dls, get_options());
+ increment = download_status_schedule_get_delay(dls, min_delay, now);
+ }
+
+ download_status_log_helper(item, !dls->increment_on, "failed",
+ "concurrently", dls->n_download_failures,
+ increment,
+ download_status_get_next_attempt_at(dls),
+ now);
+
+ if (dls->increment_on == DL_SCHED_INCREMENT_ATTEMPT) {
+ /* stop this schedule retrying on failure, it will launch concurrent
+ * connections instead */
+ return TIME_MAX;
+ } else {
+ return download_status_get_next_attempt_at(dls);
+ }
+}
+
+/** Determine when the next download attempt should be made when using an
+ * attempt-based (potentially concurrent) download schedule.
+ * Called when an attempt to download <b>dls</b> is being initiated.
+ * Increment the attempt count and set <b>dls</b>-\>next_attempt_at to an
+ * appropriate time in the future and return it.
+ * If <b>dls->increment_on</b> is DL_SCHED_INCREMENT_FAILURE, don't increment
+ * the attempts, and return a time in the far future (to avoid launching a
+ * concurrent attempt). */
+time_t
+download_status_increment_attempt(download_status_t *dls, const char *item,
+ time_t now)
+{
+ int delay = -1;
+ int min_delay = 0;
+
+ tor_assert(dls);
+
+ /* dls wasn't reset before it was used */
+ if (dls->next_attempt_at == 0) {
+ download_status_reset(dls);
+ }
+
+ if (dls->increment_on == DL_SCHED_INCREMENT_FAILURE) {
+ /* this schedule should retry on failure, and not launch any concurrent
+ attempts */
+ log_warn(LD_BUG, "Tried to launch an attempt-based connection on a "
+ "failure-based schedule.");
+ return TIME_MAX;
+ }
+
+ if (dls->n_download_attempts < IMPOSSIBLE_TO_DOWNLOAD-1)
+ ++dls->n_download_attempts;
+
+ min_delay = find_dl_min_delay(dls, get_options());
+ delay = download_status_schedule_get_delay(dls, min_delay, now);
+
+ download_status_log_helper(item, dls->increment_on, "attempted",
+ "on failure", dls->n_download_attempts,
+ delay, download_status_get_next_attempt_at(dls),
+ now);
+
+ return download_status_get_next_attempt_at(dls);
+}
+
+static time_t
+download_status_get_initial_delay_from_now(const download_status_t *dls)
+{
+ /* We use constant initial delays, even in exponential backoff
+ * schedules. */
+ return time(NULL) + find_dl_min_delay(dls, get_options());
+}
+
+/** Reset <b>dls</b> so that it will be considered downloadable
+ * immediately, and/or to show that we don't need it anymore.
+ *
+ * Must be called to initialise a download schedule, otherwise the zeroth item
+ * in the schedule will never be used.
+ *
+ * (We find the zeroth element of the download schedule, and set
+ * next_attempt_at to be the appropriate offset from 'now'. In most
+ * cases this means setting it to 'now', so the item will be immediately
+ * downloadable; when using authorities with fallbacks, there is a few seconds'
+ * delay.) */
+void
+download_status_reset(download_status_t *dls)
+{
+ if (dls->n_download_failures == IMPOSSIBLE_TO_DOWNLOAD
+ || dls->n_download_attempts == IMPOSSIBLE_TO_DOWNLOAD)
+ return; /* Don't reset this. */
+
+ dls->n_download_failures = 0;
+ dls->n_download_attempts = 0;
+ dls->next_attempt_at = download_status_get_initial_delay_from_now(dls);
+ dls->last_backoff_position = 0;
+ dls->last_delay_used = 0;
+ /* Don't reset dls->want_authority or dls->increment_on */
+}
+
+/** Return true iff, as of <b>now</b>, the resource tracked by <b>dls</b> is
+ * ready to get its download reattempted. */
+int
+download_status_is_ready(download_status_t *dls, time_t now)
+{
+ /* dls wasn't reset before it was used */
+ if (dls->next_attempt_at == 0) {
+ download_status_reset(dls);
+ }
+
+ return download_status_get_next_attempt_at(dls) <= now;
+}
+
+/** Mark <b>dl</b> as never downloadable. */
+void
+download_status_mark_impossible(download_status_t *dl)
+{
+ dl->n_download_failures = IMPOSSIBLE_TO_DOWNLOAD;
+ dl->n_download_attempts = IMPOSSIBLE_TO_DOWNLOAD;
+}
+
+/** Return the number of failures on <b>dls</b> since the last success (if
+ * any). */
+int
+download_status_get_n_failures(const download_status_t *dls)
+{
+ return dls->n_download_failures;
+}
+
+/** Return the number of attempts to download <b>dls</b> since the last success
+ * (if any). This can differ from download_status_get_n_failures() due to
+ * outstanding concurrent attempts. */
+int
+download_status_get_n_attempts(const download_status_t *dls)
+{
+ return dls->n_download_attempts;
+}
+
+/** Return the next time to attempt to download <b>dls</b>. */
+time_t
+download_status_get_next_attempt_at(const download_status_t *dls)
+{
+ /* dls wasn't reset before it was used */
+ if (dls->next_attempt_at == 0) {
+ /* so give the answer we would have given if it had been */
+ return download_status_get_initial_delay_from_now(dls);
+ }
+
+ return dls->next_attempt_at;
+}
diff --git a/src/feature/dirclient/dlstatus.h b/src/feature/dirclient/dlstatus.h
new file mode 100644
index 000000000..aeceb8cb2
--- /dev/null
+++ b/src/feature/dirclient/dlstatus.h
@@ -0,0 +1,58 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dlstatus.h
+ * \brief Header file for dlstatus.c.
+ **/
+
+#ifndef TOR_DLSTATUS_H
+#define TOR_DLSTATUS_H
+
+time_t download_status_increment_failure(download_status_t *dls,
+ int status_code, const char *item,
+ int server, time_t now);
+time_t download_status_increment_attempt(download_status_t *dls,
+ const char *item, time_t now);
+/** Increment the failure count of the download_status_t <b>dls</b>, with
+ * the optional status code <b>sc</b>. */
+#define download_status_failed(dls, sc) \
+ download_status_increment_failure((dls), (sc), NULL, \
+ dir_server_mode(get_options()), \
+ time(NULL))
+
+void download_status_reset(download_status_t *dls);
+int download_status_is_ready(download_status_t *dls, time_t now);
+time_t download_status_get_next_attempt_at(const download_status_t *dls);
+void download_status_mark_impossible(download_status_t *dl);
+
+int download_status_get_n_failures(const download_status_t *dls);
+int download_status_get_n_attempts(const download_status_t *dls);
+
+#ifdef DLSTATUS_PRIVATE
+STATIC int download_status_schedule_get_delay(download_status_t *dls,
+ int min_delay,
+ time_t now);
+
+STATIC int find_dl_min_delay(const download_status_t *dls,
+ const or_options_t *options);
+
+STATIC int next_random_exponential_delay(int delay,
+ int base_delay);
+
+STATIC void next_random_exponential_delay_range(int *low_bound_out,
+ int *high_bound_out,
+ int delay,
+ int base_delay);
+
+/* no more than quadruple the previous delay (multiplier + 1) */
+#define DIR_DEFAULT_RANDOM_MULTIPLIER (3)
+/* no more than triple the previous delay */
+#define DIR_TEST_NET_RANDOM_MULTIPLIER (2)
+
+#endif
+
+#endif /* !defined(TOR_DLSTATUS_H) */
diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c
new file mode 100644
index 000000000..8116d72c6
--- /dev/null
+++ b/src/feature/dircommon/directory.c
@@ -0,0 +1,651 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/dircache/dircache.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircommon/fp_pair.h"
+#include "feature/stats/geoip.h"
+#include "lib/compress/compress.h"
+
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+/**
+ * \file directory.c
+ * \brief Code to send and fetch information from directory authorities and
+ * caches via HTTP.
+ *
+ * Directory caches and authorities use dirserv.c to generate the results of a
+ * query and stream them to the connection; clients use routerparse.c to parse
+ * them.
+ *
+ * Every directory request has a dir_connection_t on the client side and on
+ * the server side. In most cases, the dir_connection_t object is a linked
+ * connection, tunneled through an edge_connection_t so that it can be a
+ * stream on the Tor network. The only non-tunneled connections are those
+ * that are used to upload material (descriptors and votes) to authorities.
+ * Among tunneled connections, some use one-hop circuits, and others use
+ * multi-hop circuits for anonymity.
+ *
+ * Directory requests are launched by calling
+ * directory_initiate_request(). This
+ * launch the connection, will construct an HTTP request with
+ * directory_send_command(), send the and wait for a response. The client
+ * later handles the response with connection_dir_client_reached_eof(),
+ * which passes the information received to another part of Tor.
+ *
+ * On the server side, requests are read in directory_handle_command(),
+ * which dispatches first on the request type (GET or POST), and then on
+ * the URL requested. GET requests are processed with a table-based
+ * dispatcher in url_table[]. The process of handling larger GET requests
+ * is complicated because we need to avoid allocating a copy of all the
+ * data to be sent to the client in one huge buffer. Instead, we spool the
+ * data into the buffer using logic in connection_dirserv_flushed_some() in
+ * dirserv.c. (TODO: If we extended buf.c to have a zero-copy
+ * reference-based buffer type, we could remove most of that code, at the
+ * cost of a bit more reference counting.)
+ **/
+
+/* In-points to directory.c:
+ *
+ * - directory_post_to_dirservers(), called from
+ * router_upload_dir_desc_to_dirservers() in router.c
+ * upload_service_descriptor() in rendservice.c
+ * - directory_get_from_dirserver(), called from
+ * rend_client_refetch_renddesc() in rendclient.c
+ * run_scheduled_events() in main.c
+ * do_hup() in main.c
+ * - connection_dir_process_inbuf(), called from
+ * connection_process_inbuf() in connection.c
+ * - connection_dir_finished_flushing(), called from
+ * connection_finished_flushing() in connection.c
+ * - connection_dir_finished_connecting(), called from
+ * connection_finished_connecting() in connection.c
+ */
+
+/** Convert a connection_t* to a dir_connection_t*; assert if the cast is
+ * invalid. */
+dir_connection_t *
+TO_DIR_CONN(connection_t *c)
+{
+ tor_assert(c->magic == DIR_CONNECTION_MAGIC);
+ return DOWNCAST(dir_connection_t, c);
+}
+
+/** Return false if the directory purpose <b>dir_purpose</b>
+ * does not require an anonymous (three-hop) connection.
+ *
+ * Return true 1) by default, 2) if all directory actions have
+ * specifically been configured to be over an anonymous connection,
+ * or 3) if the router is a bridge */
+int
+purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
+ const char *resource)
+{
+ if (get_options()->AllDirActionsPrivate)
+ return 1;
+
+ if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
+ if (dir_purpose == DIR_PURPOSE_FETCH_SERVERDESC
+ && resource && !strcmp(resource, "authority.z")) {
+ /* We are asking a bridge for its own descriptor. That doesn't need
+ anonymity. */
+ return 0;
+ }
+ /* Assume all other bridge stuff needs anonymity. */
+ return 1; /* if no circuits yet, this might break bootstrapping, but it's
+ * needed to be safe. */
+ }
+
+ switch (dir_purpose)
+ {
+ case DIR_PURPOSE_UPLOAD_DIR:
+ case DIR_PURPOSE_UPLOAD_VOTE:
+ case DIR_PURPOSE_UPLOAD_SIGNATURES:
+ case DIR_PURPOSE_FETCH_STATUS_VOTE:
+ case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES:
+ case DIR_PURPOSE_FETCH_CONSENSUS:
+ case DIR_PURPOSE_FETCH_CERTIFICATE:
+ case DIR_PURPOSE_FETCH_SERVERDESC:
+ case DIR_PURPOSE_FETCH_EXTRAINFO:
+ case DIR_PURPOSE_FETCH_MICRODESC:
+ return 0;
+ case DIR_PURPOSE_HAS_FETCHED_HSDESC:
+ case DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2:
+ case DIR_PURPOSE_UPLOAD_RENDDESC_V2:
+ case DIR_PURPOSE_FETCH_RENDDESC_V2:
+ case DIR_PURPOSE_FETCH_HSDESC:
+ case DIR_PURPOSE_UPLOAD_HSDESC:
+ return 1;
+ case DIR_PURPOSE_SERVER:
+ default:
+ log_warn(LD_BUG, "Called with dir_purpose=%d, router_purpose=%d",
+ dir_purpose, router_purpose);
+ tor_assert_nonfatal_unreached();
+ return 1; /* Assume it needs anonymity; better safe than sorry. */
+ }
+}
+
+/** Return a newly allocated string describing <b>auth</b>. Only describes
+ * authority features. */
+char *
+authdir_type_to_string(dirinfo_type_t auth)
+{
+ char *result;
+ smartlist_t *lst = smartlist_new();
+ if (auth & V3_DIRINFO)
+ smartlist_add(lst, (void*)"V3");
+ if (auth & BRIDGE_DIRINFO)
+ smartlist_add(lst, (void*)"Bridge");
+ if (smartlist_len(lst)) {
+ result = smartlist_join_strings(lst, ", ", 0, NULL);
+ } else {
+ result = tor_strdup("[Not an authority]");
+ }
+ smartlist_free(lst);
+ return result;
+}
+
+/** Return true iff anything we say on <b>conn</b> is being encrypted before
+ * we send it to the client/server. */
+int
+connection_dir_is_encrypted(const dir_connection_t *conn)
+{
+ /* Right now it's sufficient to see if conn is or has been linked, since
+ * the only thing it could be linked to is an edge connection on a
+ * circuit, and the only way it could have been unlinked is at the edge
+ * connection getting closed.
+ */
+ return TO_CONN(conn)->linked;
+}
+
+/** Parse an HTTP request line at the start of a headers string. On failure,
+ * return -1. On success, set *<b>command_out</b> to a copy of the HTTP
+ * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and
+ * return 0. */
+int
+parse_http_command(const char *headers, char **command_out, char **url_out)
+{
+ const char *command, *end_of_command;
+ char *s, *start, *tmp;
+
+ s = (char *)eat_whitespace_no_nl(headers);
+ if (!*s) return -1;
+ command = s;
+ s = (char *)find_whitespace(s); /* get past GET/POST */
+ if (!*s) return -1;
+ end_of_command = s;
+ s = (char *)eat_whitespace_no_nl(s);
+ if (!*s) return -1;
+ start = s; /* this is the URL, assuming it's valid */
+ s = (char *)find_whitespace(start);
+ if (!*s) return -1;
+
+ /* tolerate the http[s] proxy style of putting the hostname in the url */
+ if (s-start >= 4 && !strcmpstart(start,"http")) {
+ tmp = start + 4;
+ if (*tmp == 's')
+ tmp++;
+ if (s-tmp >= 3 && !strcmpstart(tmp,"://")) {
+ tmp = strchr(tmp+3, '/');
+ if (tmp && tmp < s) {
+ log_debug(LD_DIR,"Skipping over 'http[s]://hostname/' string");
+ start = tmp;
+ }
+ }
+ }
+
+ /* Check if the header is well formed (next sequence
+ * should be HTTP/1.X\r\n). Assumes we're supporting 1.0? */
+ {
+ unsigned minor_ver;
+ char ch;
+ char *e = (char *)eat_whitespace_no_nl(s);
+ if (2 != tor_sscanf(e, "HTTP/1.%u%c", &minor_ver, &ch)) {
+ return -1;
+ }
+ if (ch != '\r')
+ return -1;
+ }
+
+ *url_out = tor_memdup_nulterm(start, s-start);
+ *command_out = tor_memdup_nulterm(command, end_of_command - command);
+ return 0;
+}
+
+/** Return a copy of the first HTTP header in <b>headers</b> whose key is
+ * <b>which</b>. The key should be given with a terminating colon and space;
+ * this function copies everything after, up to but not including the
+ * following \\r\\n. */
+char *
+http_get_header(const char *headers, const char *which)
+{
+ const char *cp = headers;
+ while (cp) {
+ if (!strcasecmpstart(cp, which)) {
+ char *eos;
+ cp += strlen(which);
+ if ((eos = strchr(cp,'\r')))
+ return tor_strndup(cp, eos-cp);
+ else
+ return tor_strdup(cp);
+ }
+ cp = strchr(cp, '\n');
+ if (cp)
+ ++cp;
+ }
+ return NULL;
+}
+/** Parse an HTTP response string <b>headers</b> of the form
+ * \verbatim
+ * "HTTP/1.\%d \%d\%s\r\n...".
+ * \endverbatim
+ *
+ * If it's well-formed, assign the status code to *<b>code</b> and
+ * return 0. Otherwise, return -1.
+ *
+ * On success: If <b>date</b> is provided, set *date to the Date
+ * header in the http headers, or 0 if no such header is found. If
+ * <b>compression</b> is provided, set *<b>compression</b> to the
+ * compression method given in the Content-Encoding header, or 0 if no
+ * such header is found, or -1 if the value of the header is not
+ * recognized. If <b>reason</b> is provided, strdup the reason string
+ * into it.
+ */
+int
+parse_http_response(const char *headers, int *code, time_t *date,
+ compress_method_t *compression, char **reason)
+{
+ unsigned n1, n2;
+ char datestr[RFC1123_TIME_LEN+1];
+ smartlist_t *parsed_headers;
+ tor_assert(headers);
+ tor_assert(code);
+
+ while (TOR_ISSPACE(*headers)) headers++; /* tolerate leading whitespace */
+
+ if (tor_sscanf(headers, "HTTP/1.%u %u", &n1, &n2) < 2 ||
+ (n1 != 0 && n1 != 1) ||
+ (n2 < 100 || n2 >= 600)) {
+ log_warn(LD_HTTP,"Failed to parse header %s",escaped(headers));
+ return -1;
+ }
+ *code = n2;
+
+ parsed_headers = smartlist_new();
+ smartlist_split_string(parsed_headers, headers, "\n",
+ SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+ if (reason) {
+ smartlist_t *status_line_elements = smartlist_new();
+ tor_assert(smartlist_len(parsed_headers));
+ smartlist_split_string(status_line_elements,
+ smartlist_get(parsed_headers, 0),
+ " ", SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 3);
+ tor_assert(smartlist_len(status_line_elements) <= 3);
+ if (smartlist_len(status_line_elements) == 3) {
+ *reason = smartlist_get(status_line_elements, 2);
+ smartlist_set(status_line_elements, 2, NULL); /* Prevent free */
+ }
+ SMARTLIST_FOREACH(status_line_elements, char *, cp, tor_free(cp));
+ smartlist_free(status_line_elements);
+ }
+ if (date) {
+ *date = 0;
+ SMARTLIST_FOREACH(parsed_headers, const char *, s,
+ if (!strcmpstart(s, "Date: ")) {
+ strlcpy(datestr, s+6, sizeof(datestr));
+ /* This will do nothing on failure, so we don't need to check
+ the result. We shouldn't warn, since there are many other valid
+ date formats besides the one we use. */
+ parse_rfc1123_time(datestr, date);
+ break;
+ });
+ }
+ if (compression) {
+ const char *enc = NULL;
+ SMARTLIST_FOREACH(parsed_headers, const char *, s,
+ if (!strcmpstart(s, "Content-Encoding: ")) {
+ enc = s+18; break;
+ });
+
+ if (enc == NULL)
+ *compression = NO_METHOD;
+ else {
+ *compression = compression_method_get_by_name(enc);
+
+ if (*compression == UNKNOWN_METHOD)
+ log_info(LD_HTTP, "Unrecognized content encoding: %s. Trying to deal.",
+ escaped(enc));
+ }
+ }
+ SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
+ smartlist_free(parsed_headers);
+
+ return 0;
+}
+
+/** If any directory object is arriving, and it's over 10MB large, we're
+ * getting DoS'd. (As of 0.1.2.x, raw directories are about 1MB, and we never
+ * ask for more than 96 router descriptors at a time.)
+ */
+#define MAX_DIRECTORY_OBJECT_SIZE (10*(1<<20))
+
+#define MAX_VOTE_DL_SIZE (MAX_DIRECTORY_OBJECT_SIZE * 5)
+
+/** Read handler for directory connections. (That's connections <em>to</em>
+ * directory servers and connections <em>at</em> directory servers.)
+ */
+int
+connection_dir_process_inbuf(dir_connection_t *conn)
+{
+ size_t max_size;
+ tor_assert(conn);
+ tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+ /* Directory clients write, then read data until they receive EOF;
+ * directory servers read data until they get an HTTP command, then
+ * write their response (when it's finished flushing, they mark for
+ * close).
+ */
+
+ /* If we're on the dirserver side, look for a command. */
+ if (conn->base_.state == DIR_CONN_STATE_SERVER_COMMAND_WAIT) {
+ if (directory_handle_command(conn) < 0) {
+ connection_mark_for_close(TO_CONN(conn));
+ return -1;
+ }
+ return 0;
+ }
+
+ max_size =
+ (TO_CONN(conn)->purpose == DIR_PURPOSE_FETCH_STATUS_VOTE) ?
+ MAX_VOTE_DL_SIZE : MAX_DIRECTORY_OBJECT_SIZE;
+
+ if (connection_get_inbuf_len(TO_CONN(conn)) > max_size) {
+ log_warn(LD_HTTP,
+ "Too much data received from directory connection (%s): "
+ "denial of service attempt, or you need to upgrade?",
+ conn->base_.address);
+ connection_mark_for_close(TO_CONN(conn));
+ return -1;
+ }
+
+ if (!conn->base_.inbuf_reached_eof)
+ log_debug(LD_HTTP,"Got data, not eof. Leaving on inbuf.");
+ return 0;
+}
+
+/** Called when we're about to finally unlink and free a directory connection:
+ * perform necessary accounting and cleanup */
+void
+connection_dir_about_to_close(dir_connection_t *dir_conn)
+{
+ connection_t *conn = TO_CONN(dir_conn);
+
+ if (conn->state < DIR_CONN_STATE_CLIENT_FINISHED) {
+ /* It's a directory connection and connecting or fetching
+ * failed: forget about this router, and maybe try again. */
+ connection_dir_client_request_failed(dir_conn);
+ }
+
+ connection_dir_client_refetch_hsdesc_if_needed(dir_conn);
+}
+
+/** Write handler for directory connections; called when all data has
+ * been flushed. Close the connection or wait for a response as
+ * appropriate.
+ */
+int
+connection_dir_finished_flushing(dir_connection_t *conn)
+{
+ tor_assert(conn);
+ tor_assert(conn->base_.type == CONN_TYPE_DIR);
+
+ if (conn->base_.marked_for_close)
+ return 0;
+
+ /* Note that we have finished writing the directory response. For direct
+ * connections this means we're done; for tunneled connections it's only
+ * an intermediate step. */
+ if (conn->dirreq_id)
+ geoip_change_dirreq_state(conn->dirreq_id, DIRREQ_TUNNELED,
+ DIRREQ_FLUSHING_DIR_CONN_FINISHED);
+ else
+ geoip_change_dirreq_state(TO_CONN(conn)->global_identifier,
+ DIRREQ_DIRECT,
+ DIRREQ_FLUSHING_DIR_CONN_FINISHED);
+ switch (conn->base_.state) {
+ case DIR_CONN_STATE_CONNECTING:
+ case DIR_CONN_STATE_CLIENT_SENDING:
+ log_debug(LD_DIR,"client finished sending command.");
+ conn->base_.state = DIR_CONN_STATE_CLIENT_READING;
+ return 0;
+ case DIR_CONN_STATE_SERVER_WRITING:
+ if (conn->spool) {
+ log_warn(LD_BUG, "Emptied a dirserv buffer, but it's still spooling!");
+ connection_mark_for_close(TO_CONN(conn));
+ } else {
+ log_debug(LD_DIRSERV, "Finished writing server response. Closing.");
+ connection_mark_for_close(TO_CONN(conn));
+ }
+ return 0;
+ default:
+ log_warn(LD_BUG,"called in unexpected state %d.",
+ conn->base_.state);
+ tor_fragile_assert();
+ return -1;
+ }
+ return 0;
+}
+
+/** Connected handler for directory connections: begin sending data to the
+ * server, and return 0.
+ * Only used when connections don't immediately connect. */
+int
+connection_dir_finished_connecting(dir_connection_t *conn)
+{
+ tor_assert(conn);
+ tor_assert(conn->base_.type == CONN_TYPE_DIR);
+ tor_assert(conn->base_.state == DIR_CONN_STATE_CONNECTING);
+
+ log_debug(LD_HTTP,"Dir connection to router %s:%u established.",
+ conn->base_.address,conn->base_.port);
+
+ /* start flushing conn */
+ conn->base_.state = DIR_CONN_STATE_CLIENT_SENDING;
+ return 0;
+}
+
+/** Helper. Compare two fp_pair_t objects, and return negative, 0, or
+ * positive as appropriate. */
+static int
+compare_pairs_(const void **a, const void **b)
+{
+ const fp_pair_t *fp1 = *a, *fp2 = *b;
+ int r;
+ if ((r = fast_memcmp(fp1->first, fp2->first, DIGEST_LEN)))
+ return r;
+ else
+ return fast_memcmp(fp1->second, fp2->second, DIGEST_LEN);
+}
+
+/** Divide a string <b>res</b> of the form FP1-FP2+FP3-FP4...[.z], where each
+ * FP is a hex-encoded fingerprint, into a sequence of distinct sorted
+ * fp_pair_t. Skip malformed pairs. On success, return 0 and add those
+ * fp_pair_t into <b>pairs_out</b>. On failure, return -1. */
+int
+dir_split_resource_into_fingerprint_pairs(const char *res,
+ smartlist_t *pairs_out)
+{
+ smartlist_t *pairs_tmp = smartlist_new();
+ smartlist_t *pairs_result = smartlist_new();
+
+ smartlist_split_string(pairs_tmp, res, "+", 0, 0);
+ if (smartlist_len(pairs_tmp)) {
+ char *last = smartlist_get(pairs_tmp,smartlist_len(pairs_tmp)-1);
+ size_t last_len = strlen(last);
+ if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
+ last[last_len-2] = '\0';
+ }
+ }
+ SMARTLIST_FOREACH_BEGIN(pairs_tmp, char *, cp) {
+ if (strlen(cp) != HEX_DIGEST_LEN*2+1) {
+ log_info(LD_DIR,
+ "Skipping digest pair %s with non-standard length.", escaped(cp));
+ } else if (cp[HEX_DIGEST_LEN] != '-') {
+ log_info(LD_DIR,
+ "Skipping digest pair %s with missing dash.", escaped(cp));
+ } else {
+ fp_pair_t pair;
+ if (base16_decode(pair.first, DIGEST_LEN,
+ cp, HEX_DIGEST_LEN) != DIGEST_LEN ||
+ base16_decode(pair.second,DIGEST_LEN,
+ cp+HEX_DIGEST_LEN+1, HEX_DIGEST_LEN) != DIGEST_LEN) {
+ log_info(LD_DIR, "Skipping non-decodable digest pair %s", escaped(cp));
+ } else {
+ smartlist_add(pairs_result, tor_memdup(&pair, sizeof(pair)));
+ }
+ }
+ tor_free(cp);
+ } SMARTLIST_FOREACH_END(cp);
+ smartlist_free(pairs_tmp);
+
+ /* Uniq-and-sort */
+ smartlist_sort(pairs_result, compare_pairs_);
+ smartlist_uniq(pairs_result, compare_pairs_, tor_free_);
+
+ smartlist_add_all(pairs_out, pairs_result);
+ smartlist_free(pairs_result);
+ return 0;
+}
+
+/** Given a directory <b>resource</b> request, containing zero
+ * or more strings separated by plus signs, followed optionally by ".z", store
+ * the strings, in order, into <b>fp_out</b>. If <b>compressed_out</b> is
+ * non-NULL, set it to 1 if the resource ends in ".z", else set it to 0.
+ *
+ * If (flags & DSR_HEX), then delete all elements that aren't hex digests, and
+ * decode the rest. If (flags & DSR_BASE64), then use "-" rather than "+" as
+ * a separator, delete all the elements that aren't base64-encoded digests,
+ * and decode the rest. If (flags & DSR_DIGEST256), these digests should be
+ * 256 bits long; else they should be 160.
+ *
+ * If (flags & DSR_SORT_UNIQ), then sort the list and remove all duplicates.
+ */
+int
+dir_split_resource_into_fingerprints(const char *resource,
+ smartlist_t *fp_out, int *compressed_out,
+ int flags)
+{
+ const int decode_hex = flags & DSR_HEX;
+ const int decode_base64 = flags & DSR_BASE64;
+ const int digests_are_256 = flags & DSR_DIGEST256;
+ const int sort_uniq = flags & DSR_SORT_UNIQ;
+
+ const int digest_len = digests_are_256 ? DIGEST256_LEN : DIGEST_LEN;
+ const int hex_digest_len = digests_are_256 ?
+ HEX_DIGEST256_LEN : HEX_DIGEST_LEN;
+ const int base64_digest_len = digests_are_256 ?
+ BASE64_DIGEST256_LEN : BASE64_DIGEST_LEN;
+ smartlist_t *fp_tmp = smartlist_new();
+
+ tor_assert(!(decode_hex && decode_base64));
+ tor_assert(fp_out);
+
+ smartlist_split_string(fp_tmp, resource, decode_base64?"-":"+", 0, 0);
+ if (compressed_out)
+ *compressed_out = 0;
+ if (smartlist_len(fp_tmp)) {
+ char *last = smartlist_get(fp_tmp,smartlist_len(fp_tmp)-1);
+ size_t last_len = strlen(last);
+ if (last_len > 2 && !strcmp(last+last_len-2, ".z")) {
+ last[last_len-2] = '\0';
+ if (compressed_out)
+ *compressed_out = 1;
+ }
+ }
+ if (decode_hex || decode_base64) {
+ const size_t encoded_len = decode_hex ? hex_digest_len : base64_digest_len;
+ int i;
+ char *cp, *d = NULL;
+ for (i = 0; i < smartlist_len(fp_tmp); ++i) {
+ cp = smartlist_get(fp_tmp, i);
+ if (strlen(cp) != encoded_len) {
+ log_info(LD_DIR,
+ "Skipping digest %s with non-standard length.", escaped(cp));
+ smartlist_del_keeporder(fp_tmp, i--);
+ goto again;
+ }
+ d = tor_malloc_zero(digest_len);
+ if (decode_hex ?
+ (base16_decode(d, digest_len, cp, hex_digest_len) != digest_len) :
+ (base64_decode(d, digest_len, cp, base64_digest_len)
+ != digest_len)) {
+ log_info(LD_DIR, "Skipping non-decodable digest %s", escaped(cp));
+ smartlist_del_keeporder(fp_tmp, i--);
+ goto again;
+ }
+ smartlist_set(fp_tmp, i, d);
+ d = NULL;
+ again:
+ tor_free(cp);
+ tor_free(d);
+ }
+ }
+ if (sort_uniq) {
+ if (decode_hex || decode_base64) {
+ if (digests_are_256) {
+ smartlist_sort_digests256(fp_tmp);
+ smartlist_uniq_digests256(fp_tmp);
+ } else {
+ smartlist_sort_digests(fp_tmp);
+ smartlist_uniq_digests(fp_tmp);
+ }
+ } else {
+ smartlist_sort_strings(fp_tmp);
+ smartlist_uniq_strings(fp_tmp);
+ }
+ }
+ smartlist_add_all(fp_out, fp_tmp);
+ smartlist_free(fp_tmp);
+ return 0;
+}
+
+/** As dir_split_resource_into_fingerprints, but instead fills
+ * <b>spool_out</b> with a list of spoolable_resource_t for the resource
+ * identified through <b>source</b>. */
+int
+dir_split_resource_into_spoolable(const char *resource,
+ dir_spool_source_t source,
+ smartlist_t *spool_out,
+ int *compressed_out,
+ int flags)
+{
+ smartlist_t *fingerprints = smartlist_new();
+
+ tor_assert(flags & (DSR_HEX|DSR_BASE64));
+ const size_t digest_len =
+ (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN;
+
+ int r = dir_split_resource_into_fingerprints(resource, fingerprints,
+ compressed_out, flags);
+ /* This is not a very efficient implementation XXXX */
+ SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) {
+ spooled_resource_t *spooled =
+ spooled_resource_new(source, digest, digest_len);
+ if (spooled)
+ smartlist_add(spool_out, spooled);
+ tor_free(digest);
+ } SMARTLIST_FOREACH_END(digest);
+
+ smartlist_free(fingerprints);
+ return r;
+}
diff --git a/src/feature/dircommon/directory.h b/src/feature/dircommon/directory.h
new file mode 100644
index 000000000..ec95573f5
--- /dev/null
+++ b/src/feature/dircommon/directory.h
@@ -0,0 +1,129 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file directory.h
+ * \brief Header file for directory.c.
+ **/
+
+#ifndef TOR_DIRECTORY_H
+#define TOR_DIRECTORY_H
+
+dir_connection_t *TO_DIR_CONN(connection_t *c);
+
+#define DIR_CONN_STATE_MIN_ 1
+/** State for connection to directory server: waiting for connect(). */
+#define DIR_CONN_STATE_CONNECTING 1
+/** State for connection to directory server: sending HTTP request. */
+#define DIR_CONN_STATE_CLIENT_SENDING 2
+/** State for connection to directory server: reading HTTP response. */
+#define DIR_CONN_STATE_CLIENT_READING 3
+/** State for connection to directory server: happy and finished. */
+#define DIR_CONN_STATE_CLIENT_FINISHED 4
+/** State for connection at directory server: waiting for HTTP request. */
+#define DIR_CONN_STATE_SERVER_COMMAND_WAIT 5
+/** State for connection at directory server: sending HTTP response. */
+#define DIR_CONN_STATE_SERVER_WRITING 6
+#define DIR_CONN_STATE_MAX_ 6
+
+#define DIR_PURPOSE_MIN_ 4
+/** A connection to a directory server: set after a v2 rendezvous
+ * descriptor is downloaded. */
+#define DIR_PURPOSE_HAS_FETCHED_RENDDESC_V2 4
+/** A connection to a directory server: download one or more server
+ * descriptors. */
+#define DIR_PURPOSE_FETCH_SERVERDESC 6
+/** A connection to a directory server: download one or more extra-info
+ * documents. */
+#define DIR_PURPOSE_FETCH_EXTRAINFO 7
+/** A connection to a directory server: upload a server descriptor. */
+#define DIR_PURPOSE_UPLOAD_DIR 8
+/** A connection to a directory server: upload a v3 networkstatus vote. */
+#define DIR_PURPOSE_UPLOAD_VOTE 10
+/** A connection to a directory server: upload a v3 consensus signature */
+#define DIR_PURPOSE_UPLOAD_SIGNATURES 11
+/** A connection to a directory server: download one or more v3 networkstatus
+ * votes. */
+#define DIR_PURPOSE_FETCH_STATUS_VOTE 12
+/** A connection to a directory server: download a v3 detached signatures
+ * object for a consensus. */
+#define DIR_PURPOSE_FETCH_DETACHED_SIGNATURES 13
+/** A connection to a directory server: download a v3 networkstatus
+ * consensus. */
+#define DIR_PURPOSE_FETCH_CONSENSUS 14
+/** A connection to a directory server: download one or more directory
+ * authority certificates. */
+#define DIR_PURPOSE_FETCH_CERTIFICATE 15
+
+/** Purpose for connection at a directory server. */
+#define DIR_PURPOSE_SERVER 16
+/** A connection to a hidden service directory server: upload a v2 rendezvous
+ * descriptor. */
+#define DIR_PURPOSE_UPLOAD_RENDDESC_V2 17
+/** A connection to a hidden service directory server: download a v2 rendezvous
+ * descriptor. */
+#define DIR_PURPOSE_FETCH_RENDDESC_V2 18
+/** A connection to a directory server: download a microdescriptor. */
+#define DIR_PURPOSE_FETCH_MICRODESC 19
+/** A connection to a hidden service directory: upload a v3 descriptor. */
+#define DIR_PURPOSE_UPLOAD_HSDESC 20
+/** A connection to a hidden service directory: fetch a v3 descriptor. */
+#define DIR_PURPOSE_FETCH_HSDESC 21
+/** A connection to a directory server: set after a hidden service descriptor
+ * is downloaded. */
+#define DIR_PURPOSE_HAS_FETCHED_HSDESC 22
+#define DIR_PURPOSE_MAX_ 22
+
+/** True iff <b>p</b> is a purpose corresponding to uploading
+ * data to a directory server. */
+#define DIR_PURPOSE_IS_UPLOAD(p) \
+ ((p)==DIR_PURPOSE_UPLOAD_DIR || \
+ (p)==DIR_PURPOSE_UPLOAD_VOTE || \
+ (p)==DIR_PURPOSE_UPLOAD_SIGNATURES || \
+ (p)==DIR_PURPOSE_UPLOAD_RENDDESC_V2 || \
+ (p)==DIR_PURPOSE_UPLOAD_HSDESC)
+
+enum compress_method_t;
+int parse_http_response(const char *headers, int *code, time_t *date,
+ enum compress_method_t *compression, char **response);
+int parse_http_command(const char *headers,
+ char **command_out, char **url_out);
+char *http_get_header(const char *headers, const char *which);
+
+int connection_dir_is_encrypted(const dir_connection_t *conn);
+int connection_dir_reached_eof(dir_connection_t *conn);
+int connection_dir_process_inbuf(dir_connection_t *conn);
+int connection_dir_finished_flushing(dir_connection_t *conn);
+int connection_dir_finished_connecting(dir_connection_t *conn);
+void connection_dir_about_to_close(dir_connection_t *dir_conn);
+
+#define DSR_HEX (1<<0)
+#define DSR_BASE64 (1<<1)
+#define DSR_DIGEST256 (1<<2)
+#define DSR_SORT_UNIQ (1<<3)
+int dir_split_resource_into_fingerprints(const char *resource,
+ smartlist_t *fp_out, int *compressed_out,
+ int flags);
+enum dir_spool_source_t;
+int dir_split_resource_into_spoolable(const char *resource,
+ enum dir_spool_source_t source,
+ smartlist_t *spool_out,
+ int *compressed_out,
+ int flags);
+int dir_split_resource_into_fingerprint_pairs(const char *res,
+ smartlist_t *pairs_out);
+char *directory_dump_request_log(void);
+void note_request(const char *key, size_t bytes);
+
+int purpose_needs_anonymity(uint8_t dir_purpose, uint8_t router_purpose,
+ const char *resource);
+
+char *authdir_type_to_string(dirinfo_type_t auth);
+
+#define X_ADDRESS_HEADER "X-Your-Address-Is: "
+#define X_OR_DIFF_FROM_CONSENSUS_HEADER "X-Or-Diff-From-Consensus: "
+
+#endif /* !defined(TOR_DIRECTORY_H) */
diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c
index 563f6c6b3..1ba0428c5 100644
--- a/src/feature/hs/hs_client.c
+++ b/src/feature/hs/hs_client.c
@@ -19,7 +19,8 @@
#include "lib/crypt_ops/crypto_format.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
#include "feature/hs/hs_cache.h"
#include "feature/hs/hs_cell.h"
#include "feature/hs/hs_circuit.h"
diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c
index 643240fb6..dc7bb41ee 100644
--- a/src/feature/hs/hs_service.c
+++ b/src/feature/hs/hs_service.c
@@ -19,7 +19,8 @@
#include "lib/crypt_ops/crypto_util.h"
#include "lib/crypt_ops/crypto_ope.h"
#include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
#include "core/mainloop/mainloop.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/nodelist.h"
diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c
index 74a6c5b5f..e070cb142 100644
--- a/src/feature/nodelist/authcert.c
+++ b/src/feature/nodelist/authcert.c
@@ -23,7 +23,9 @@
#include "core/mainloop/mainloop.h"
#include "core/or/policies.h"
#include "feature/client/bridges.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
#include "feature/dircommon/fp_pair.h"
#include "feature/nodelist/authcert.h"
#include "feature/nodelist/dirlist.h"
diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c
index 8f78df521..0935145d8 100644
--- a/src/feature/nodelist/dirlist.c
+++ b/src/feature/nodelist/dirlist.c
@@ -29,7 +29,7 @@
#include "app/config/config.h"
#include "core/or/policies.h"
#include "feature/control/control.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/nodelist/nodelist.h"
diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c
index 203d079b1..96696b603 100644
--- a/src/feature/nodelist/microdesc.c
+++ b/src/feature/nodelist/microdesc.c
@@ -14,8 +14,9 @@
#include "core/or/circuitbuild.h"
#include "app/config/config.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dlstatus.h"
#include "feature/client/entrynodes.h"
#include "feature/nodelist/microdesc.h"
#include "feature/nodelist/networkstatus.h"
diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c
index c64c5176e..c6a51aefc 100644
--- a/src/feature/nodelist/networkstatus.c
+++ b/src/feature/nodelist/networkstatus.c
@@ -51,7 +51,9 @@
#include "feature/control/control.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
#include "feature/dircache/dirserv.h"
#include "feature/dirauth/reachability.h"
#include "core/or/dos.h"
diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c
index a9f045d45..aab0bbe9f 100644
--- a/src/feature/nodelist/node_select.c
+++ b/src/feature/nodelist/node_select.c
@@ -18,7 +18,8 @@
#include "core/or/policies.h"
#include "core/or/reasons.h"
#include "feature/client/entrynodes.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
#include "feature/nodelist/dirlist.h"
#include "feature/nodelist/microdesc.h"
#include "feature/nodelist/networkstatus.h"
diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c
index 9a4e48b07..5c6a10477 100644
--- a/src/feature/nodelist/routerlist.c
+++ b/src/feature/nodelist/routerlist.c
@@ -69,8 +69,10 @@
#include "feature/client/bridges.h"
#include "feature/control/control.h"
#include "feature/dirauth/mode.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dlstatus.h"
#include "feature/dirauth/reachability.h"
#include "feature/dirauth/process_descs.h"
#include "feature/nodelist/authcert.h"
diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c
index 447c21e89..622cfeb86 100644
--- a/src/feature/relay/router.c
+++ b/src/feature/relay/router.c
@@ -16,7 +16,8 @@
#include "lib/crypt_ops/crypto_rand.h"
#include "lib/crypt_ops/crypto_util.h"
#include "lib/crypt_ops/crypto_curve25519.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
#include "feature/dircache/dirserv.h"
#include "feature/dirauth/process_descs.h"
#include "feature/relay/dns.h"
diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c
index a2b494bd2..d1310699f 100644
--- a/src/feature/rend/rendclient.c
+++ b/src/feature/rend/rendclient.c
@@ -19,7 +19,8 @@
#include "lib/crypt_ops/crypto_dh.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
#include "feature/hs/hs_circuit.h"
#include "feature/hs/hs_client.h"
#include "feature/hs/hs_common.h"
diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c
index ceb4d958d..7966ab846 100644
--- a/src/feature/rend/rendservice.c
+++ b/src/feature/rend/rendservice.c
@@ -19,7 +19,8 @@
#include "lib/crypt_ops/crypto_dh.h"
#include "lib/crypt_ops/crypto_rand.h"
#include "lib/crypt_ops/crypto_util.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
#include "feature/hs/hs_common.h"
#include "feature/hs/hs_config.h"
#include "core/mainloop/mainloop.h"
diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c
index 2fbb27561..06483282b 100644
--- a/src/test/fuzz/fuzz_http.c
+++ b/src/test/fuzz/fuzz_http.c
@@ -4,14 +4,14 @@
#include "orconfig.h"
#define BUFFERS_PRIVATE
-#define DIRECTORY_PRIVATE
+#define DIRCACHE_PRIVATE
#include "core/or/or.h"
#include "lib/err/backtrace.h"
#include "lib/container/buffers.h"
#include "app/config/config.h"
#include "core/mainloop/connection.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircache/dircache.h"
#include "lib/log/log.h"
#include "feature/dircommon/dir_connection_st.h"
@@ -132,4 +132,3 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size)
return 0;
}
-
diff --git a/src/test/test_connection.c b/src/test/test_connection.c
index d85ede61b..0013f47fb 100644
--- a/src/test/test_connection.c
+++ b/src/test/test_connection.c
@@ -18,7 +18,7 @@
#include "feature/nodelist/nodelist.h"
#include "feature/nodelist/networkstatus.h"
#include "feature/rend/rendcache.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "core/or/connection_or.h"
#include "lib/net/resolve.h"
diff --git a/src/test/test_dir.c b/src/test/test_dir.c
index b2c3f4426..686b03949 100644
--- a/src/test/test_dir.c
+++ b/src/test/test_dir.c
@@ -9,8 +9,11 @@
#define BWAUTH_PRIVATE
#define CONFIG_PRIVATE
#define CONTROL_PRIVATE
+#define DIRCACHE_PRIVATE
+#define DIRCLIENT_PRIVATE
#define DIRSERV_PRIVATE
#define DIRVOTE_PRIVATE
+#define DLSTATUS_PRIVATE
#define HIBERNATE_PRIVATE
#define NETWORKSTATUS_PRIVATE
#define NODE_SELECT_PRIVATE
@@ -30,8 +33,11 @@
#include "lib/crypt_ops/crypto_ed25519.h"
#include "lib/crypt_ops/crypto_format.h"
#include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
#include "feature/dirauth/bwauth.h"
+#include "feature/dircache/dircache.h"
#include "feature/dircache/dirserv.h"
#include "feature/dirauth/process_descs.h"
#include "feature/dirauth/dirvote.h"
diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c
index 2cfed16b5..f05b0fc79 100644
--- a/src/test/test_dir_handle_get.c
+++ b/src/test/test_dir_handle_get.c
@@ -8,12 +8,14 @@
#define CONNECTION_PRIVATE
#define CONFIG_PRIVATE
#define RENDCACHE_PRIVATE
+#define DIRCACHE_PRIVATE
#include "core/or/or.h"
#include "app/config/config.h"
#include "core/mainloop/connection.h"
#include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dircache/dircache.h"
#include "test/test.h"
#include "lib/compress/compress.h"
#include "feature/rend/rendcommon.h"
diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c
index cb694106c..ebe9accf5 100644
--- a/src/test/test_entrynodes.c
+++ b/src/test/test_entrynodes.c
@@ -8,7 +8,7 @@
#define STATEFILE_PRIVATE
#define ENTRYNODES_PRIVATE
#define ROUTERLIST_PRIVATE
-#define DIRECTORY_PRIVATE
+#define DIRCLIENT_PRIVATE
#include "core/or/or.h"
#include "test/test.h"
@@ -19,7 +19,8 @@
#include "app/config/config.h"
#include "app/config/confparse.h"
#include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
#include "feature/client/entrynodes.h"
#include "feature/nodelist/nodelist.h"
#include "feature/nodelist/networkstatus.h"
diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c
index 728bb4a2f..203a1d703 100644
--- a/src/test/test_hs_cache.c
+++ b/src/test/test_hs_cache.c
@@ -7,13 +7,15 @@
*/
#define CONNECTION_PRIVATE
-#define DIRECTORY_PRIVATE
+#define DIRCACHE_PRIVATE
+#define DIRCLIENT_PRIVATE
#define HS_CACHE_PRIVATE
#include "trunnel/ed25519_cert.h"
#include "feature/hs/hs_cache.h"
#include "feature/rend/rendcache.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircache/dircache.h"
+#include "feature/dirclient/dirclient.h"
#include "feature/nodelist/networkstatus.h"
#include "core/mainloop/connection.h"
#include "core/proto/proto_http.h"
diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c
index 31a2816c1..25cb991a7 100644
--- a/src/test/test_hs_client.c
+++ b/src/test/test_hs_client.c
@@ -25,7 +25,7 @@
#include "lib/crypt_ops/crypto_cipher.h"
#include "lib/crypt_ops/crypto_dh.h"
#include "core/or/channeltls.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "core/mainloop/mainloop.h"
#include "feature/nodelist/nodelist.h"
#include "feature/nodelist/routerset.h"
diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c
index c60d6e264..95f7ed14b 100644
--- a/src/test/test_hs_common.c
+++ b/src/test/test_hs_common.c
@@ -24,7 +24,7 @@
#include "feature/hs/hs_service.h"
#include "app/config/config.h"
#include "feature/nodelist/networkstatus.h"
-#include "feature/dircache/directory.h"
+#include "feature/dirclient/dirclient.h"
#include "feature/dirauth/dirvote.h"
#include "feature/nodelist/nodelist.h"
#include "feature/nodelist/routerlist.h"
diff --git a/src/test/test_oos.c b/src/test/test_oos.c
index b8ff3030a..fb0daa7a8 100644
--- a/src/test/test_oos.c
+++ b/src/test/test_oos.c
@@ -9,7 +9,7 @@
#include "app/config/config.h"
#include "core/mainloop/connection.h"
#include "core/or/connection_or.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
#include "core/mainloop/mainloop.h"
#include "test/test.h"
diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c
index f629596c5..1086b9861 100644
--- a/src/test/test_routerlist.c
+++ b/src/test/test_routerlist.c
@@ -6,7 +6,7 @@
#include <time.h>
#define CONNECTION_PRIVATE
-#define DIRECTORY_PRIVATE
+#define DIRCLIENT_PRIVATE
#define DIRVOTE_PRIVATE
#define ENTRYNODES_PRIVATE
#define HIBERNATE_PRIVATE
@@ -19,7 +19,8 @@
#include "core/mainloop/connection.h"
#include "feature/control/control.h"
#include "lib/crypt_ops/crypto_rand.h"
-#include "feature/dircache/directory.h"
+#include "feature/dircommon/directory.h"
+#include "feature/dirclient/dirclient.h"
#include "feature/dirauth/dirvote.h"
#include "feature/client/entrynodes.h"
#include "feature/hibernate/hibernate.h"
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits