[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [tor/master] Merge branch 'bug24104_029_squashed'
commit 94b04d6c64ec998a9117d65a156888fa3af188e5
Merge: 1c62adb65 81f422332
Author: Nick Mathewson <nickm@xxxxxxxxxxxxxx>
Date: Tue Sep 4 10:44:36 2018 -0400
Merge branch 'bug24104_029_squashed'
changes/bug24104 | 4 ++
src/feature/relay/router.c | 27 +++++++--
src/feature/stats/rephist.c | 4 +-
src/feature/stats/rephist.h | 2 +-
src/test/log_test_helpers.c | 20 +++++++
src/test/log_test_helpers.h | 5 ++
src/test/test_router.c | 130 ++++++++++++++++++++++++++++++++++++++++++--
7 files changed, 180 insertions(+), 12 deletions(-)
diff --cc src/feature/relay/router.c
index ad97d534c,000000000..b9a930dbe
mode 100644,000000..100644
--- a/src/feature/relay/router.c
+++ b/src/feature/relay/router.c
@@@ -1,3879 -1,0 +1,3896 @@@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define ROUTER_PRIVATE
+
+#include "core/or/or.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/control/control.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "feature/dircache/directory.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/relay/dns.h"
+#include "feature/stats/geoip.h"
+#include "feature/hibernate/hibernate.h"
+#include "core/mainloop/main.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "core/or/policies.h"
+#include "core/or/protover.h"
+#include "core/or/relay.h"
+#include "feature/stats/rephist.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerparse.h"
+#include "app/config/statefile.h"
+#include "feature/nodelist/torcert.h"
+#include "feature/client/transports.h"
+#include "feature/nodelist/routerset.h"
+
+#include "feature/dirauth/mode.h"
+
+#include "feature/nodelist/authority_cert_st.h"
+#include "core/or/crypt_path_st.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/dirclient/dir_server_st.h"
+#include "core/or/extend_info_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "app/config/or_state_st.h"
+#include "core/or/port_cfg_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+#include "lib/osinfo/uname.h"
+#include "lib/tls/tortls.h"
+#include "lib/encoding/confline.h"
+#include "lib/crypt_ops/crypto_format.h"
+
+/**
+ * \file router.c
+ * \brief Miscellaneous relay functionality, including RSA key maintenance,
+ * generating and uploading server descriptors, picking an address to
+ * advertise, and so on.
+ *
+ * This module handles the job of deciding whether we are a Tor relay, and if
+ * so what kind. (Mostly through functions like server_mode() that inspect an
+ * or_options_t, but in some cases based on our own capabilities, such as when
+ * we are deciding whether to be a directory cache in
+ * router_has_bandwidth_to_be_dirserver().)
+ *
+ * Also in this module are the functions to generate our own routerinfo_t and
+ * extrainfo_t, and to encode those to signed strings for upload to the
+ * directory authorities.
+ *
+ * This module also handles key maintenance for RSA and Curve25519-ntor keys,
+ * and for our TLS context. (These functions should eventually move to
+ * routerkeys.c along with the code that handles Ed25519 keys now.)
+ **/
+
+/************************************************************/
+
+/*****
+ * Key management: ORs only.
+ *****/
+
+/** Private keys for this OR. There is also an SSL key managed by tortls.c.
+ */
+static tor_mutex_t *key_lock=NULL;
+static time_t onionkey_set_at=0; /**< When was onionkey last changed? */
+/** Current private onionskin decryption key: used to decode CREATE cells. */
+static crypto_pk_t *onionkey=NULL;
+/** Previous private onionskin decryption key: used to decode CREATE cells
+ * generated by clients that have an older version of our descriptor. */
+static crypto_pk_t *lastonionkey=NULL;
+/** Current private ntor secret key: used to perform the ntor handshake. */
+static curve25519_keypair_t curve25519_onion_key;
+/** Previous private ntor secret key: used to perform the ntor handshake
+ * with clients that have an older version of our descriptor. */
+static curve25519_keypair_t last_curve25519_onion_key;
+/** Private server "identity key": used to sign directory info and TLS
+ * certificates. Never changes. */
+static crypto_pk_t *server_identitykey=NULL;
+/** Digest of server_identitykey. */
+static char server_identitykey_digest[DIGEST_LEN];
+/** Private client "identity key": used to sign bridges' and clients'
+ * outbound TLS certificates. Regenerated on startup and on IP address
+ * change. */
+static crypto_pk_t *client_identitykey=NULL;
+/** Signing key used for v3 directory material; only set for authorities. */
+static crypto_pk_t *authority_signing_key = NULL;
+/** Key certificate to authenticate v3 directory material; only set for
+ * authorities. */
+static authority_cert_t *authority_key_certificate = NULL;
+
+/** For emergency V3 authority key migration: An extra signing key that we use
+ * with our old (obsolete) identity key for a while. */
+static crypto_pk_t *legacy_signing_key = NULL;
+/** For emergency V3 authority key migration: An extra certificate to
+ * authenticate legacy_signing_key with our obsolete identity key.*/
+static authority_cert_t *legacy_key_certificate = NULL;
+
+/* (Note that v3 authorities also have a separate "authority identity key",
+ * but this key is never actually loaded by the Tor process. Instead, it's
+ * used by tor-gencert to sign new signing keys and make new key
+ * certificates. */
+
+const char *format_node_description(char *buf,
+ const char *id_digest,
+ int is_named,
+ const char *nickname,
+ const tor_addr_t *addr,
+ uint32_t addr32h);
+
+/** Return a readonly string with human readable description
+ * of <b>err</b>.
+ */
+const char *
+routerinfo_err_to_string(int err)
+{
+ switch (err) {
+ case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR:
+ return "No known exit address yet";
+ case TOR_ROUTERINFO_ERROR_CANNOT_PARSE:
+ return "Cannot parse descriptor";
+ case TOR_ROUTERINFO_ERROR_NOT_A_SERVER:
+ return "Not running in server mode";
+ case TOR_ROUTERINFO_ERROR_DIGEST_FAILED:
+ return "Key digest failed";
+ case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE:
+ return "Cannot generate descriptor";
+ case TOR_ROUTERINFO_ERROR_DESC_REBUILDING:
+ return "Descriptor still rebuilding - not ready yet";
+ }
+
+ log_warn(LD_BUG, "unknown routerinfo error %d - shouldn't happen", err);
+ tor_assert_unreached();
+
+ return "Unknown error";
+}
+
+/** Return true if we expect given error to be transient.
+ * Return false otherwise.
+ */
+int
+routerinfo_err_is_transient(int err)
+{
+ /**
+ * For simplicity, we consider all errors other than
+ * "not a server" transient - see discussion on
+ * https://trac.torproject.org/projects/tor/ticket/27034
+ */
+ return err != TOR_ROUTERINFO_ERROR_NOT_A_SERVER;
+}
+
+/** Replace the current onion key with <b>k</b>. Does not affect
+ * lastonionkey; to update lastonionkey correctly, call rotate_onion_key().
+ */
+static void
+set_onion_key(crypto_pk_t *k)
+{
+ if (onionkey && crypto_pk_eq_keys(onionkey, k)) {
+ /* k is already our onion key; free it and return */
+ crypto_pk_free(k);
+ return;
+ }
+ tor_mutex_acquire(key_lock);
+ crypto_pk_free(onionkey);
+ onionkey = k;
+ tor_mutex_release(key_lock);
+ mark_my_descriptor_dirty("set onion key");
+}
+
+/** Return the current onion key. Requires that the onion key has been
+ * loaded or generated. */
+crypto_pk_t *
+get_onion_key(void)
+{
+ tor_assert(onionkey);
+ return onionkey;
+}
+
+/** Store a full copy of the current onion key into *<b>key</b>, and a full
+ * copy of the most recent onion key into *<b>last</b>. Store NULL into
+ * a pointer if the corresponding key does not exist.
+ */
+void
+dup_onion_keys(crypto_pk_t **key, crypto_pk_t **last)
+{
+ tor_assert(key);
+ tor_assert(last);
+ tor_mutex_acquire(key_lock);
+ if (onionkey)
+ *key = crypto_pk_copy_full(onionkey);
+ else
+ *key = NULL;
+ if (lastonionkey)
+ *last = crypto_pk_copy_full(lastonionkey);
+ else
+ *last = NULL;
+ tor_mutex_release(key_lock);
+}
+
+/** Expire our old set of onion keys. This is done by setting
+ * last_curve25519_onion_key and lastonionkey to all zero's and NULL
+ * respectively.
+ *
+ * This function does not perform any grace period checks for the old onion
+ * keys.
+ */
+void
+expire_old_onion_keys(void)
+{
+ char *fname = NULL;
+
+ tor_mutex_acquire(key_lock);
+
+ /* Free lastonionkey and set it to NULL. */
+ if (lastonionkey) {
+ crypto_pk_free(lastonionkey);
+ lastonionkey = NULL;
+ }
+
+ /* We zero out the keypair. See the tor_mem_is_zero() check made in
+ * construct_ntor_key_map() below. */
+ memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
+
+ tor_mutex_release(key_lock);
+
+ fname = get_keydir_fname("secret_onion_key.old");
+ if (file_status(fname) == FN_FILE) {
+ if (tor_unlink(fname) != 0) {
+ log_warn(LD_FS, "Couldn't unlink old onion key file %s: %s",
+ fname, strerror(errno));
+ }
+ }
+ tor_free(fname);
+
+ fname = get_keydir_fname("secret_onion_key_ntor.old");
+ if (file_status(fname) == FN_FILE) {
+ if (tor_unlink(fname) != 0) {
+ log_warn(LD_FS, "Couldn't unlink old ntor onion key file %s: %s",
+ fname, strerror(errno));
+ }
+ }
+ tor_free(fname);
+}
+
+/** Return the current secret onion key for the ntor handshake. Must only
+ * be called from the main thread. */
+static const curve25519_keypair_t *
+get_current_curve25519_keypair(void)
+{
+ return &curve25519_onion_key;
+}
+/** Return a map from KEYID (the key itself) to keypairs for use in the ntor
+ * handshake. Must only be called from the main thread. */
+di_digest256_map_t *
+construct_ntor_key_map(void)
+{
+ di_digest256_map_t *m = NULL;
+
+ if (!tor_mem_is_zero((const char*)
+ curve25519_onion_key.pubkey.public_key,
+ CURVE25519_PUBKEY_LEN)) {
+ dimap_add_entry(&m,
+ curve25519_onion_key.pubkey.public_key,
+ tor_memdup(&curve25519_onion_key,
+ sizeof(curve25519_keypair_t)));
+ }
+ if (!tor_mem_is_zero((const char*)
+ last_curve25519_onion_key.pubkey.public_key,
+ CURVE25519_PUBKEY_LEN)) {
+ dimap_add_entry(&m,
+ last_curve25519_onion_key.pubkey.public_key,
+ tor_memdup(&last_curve25519_onion_key,
+ sizeof(curve25519_keypair_t)));
+ }
+
+ return m;
+}
+/** Helper used to deallocate a di_digest256_map_t returned by
+ * construct_ntor_key_map. */
+static void
+ntor_key_map_free_helper(void *arg)
+{
+ curve25519_keypair_t *k = arg;
+ memwipe(k, 0, sizeof(*k));
+ tor_free(k);
+}
+/** Release all storage from a keymap returned by construct_ntor_key_map. */
+void
+ntor_key_map_free_(di_digest256_map_t *map)
+{
+ if (!map)
+ return;
+ dimap_free(map, ntor_key_map_free_helper);
+}
+
+/** Return the time when the onion key was last set. This is either the time
+ * when the process launched, or the time of the most recent key rotation since
+ * the process launched.
+ */
+time_t
+get_onion_key_set_at(void)
+{
+ return onionkey_set_at;
+}
+
+/** Set the current server identity key to <b>k</b>.
+ */
+void
+set_server_identity_key(crypto_pk_t *k)
+{
+ crypto_pk_free(server_identitykey);
+ server_identitykey = k;
+ if (crypto_pk_get_digest(server_identitykey,
+ server_identitykey_digest) < 0) {
+ log_err(LD_BUG, "Couldn't compute our own identity key digest.");
+ tor_assert(0);
+ }
+}
+
+/** Make sure that we have set up our identity keys to match or not match as
+ * appropriate, and die with an assertion if we have not. */
+static void
+assert_identity_keys_ok(void)
+{
+ if (1)
+ return;
+ tor_assert(client_identitykey);
+ if (public_server_mode(get_options())) {
+ /* assert that we have set the client and server keys to be equal */
+ tor_assert(server_identitykey);
+ tor_assert(crypto_pk_eq_keys(client_identitykey, server_identitykey));
+ } else {
+ /* assert that we have set the client and server keys to be unequal */
+ if (server_identitykey)
+ tor_assert(!crypto_pk_eq_keys(client_identitykey, server_identitykey));
+ }
+}
+
+/** Returns the current server identity key; requires that the key has
+ * been set, and that we are running as a Tor server.
+ */
+crypto_pk_t *
+get_server_identity_key(void)
+{
+ tor_assert(server_identitykey);
+ tor_assert(server_mode(get_options()));
+ assert_identity_keys_ok();
+ return server_identitykey;
+}
+
+/** Return true iff we are a server and the server identity key
+ * has been set. */
+int
+server_identity_key_is_set(void)
+{
+ return server_mode(get_options()) && server_identitykey != NULL;
+}
+
+/** Set the current client identity key to <b>k</b>.
+ */
+void
+set_client_identity_key(crypto_pk_t *k)
+{
+ crypto_pk_free(client_identitykey);
+ client_identitykey = k;
+}
+
+/** Returns the current client identity key for use on outgoing TLS
+ * connections; requires that the key has been set.
+ */
+crypto_pk_t *
+get_tlsclient_identity_key(void)
+{
+ tor_assert(client_identitykey);
+ assert_identity_keys_ok();
+ return client_identitykey;
+}
+
+/** Return true iff the client identity key has been set. */
+int
+client_identity_key_is_set(void)
+{
+ return client_identitykey != NULL;
+}
+
+/** Return the key certificate for this v3 (voting) authority, or NULL
+ * if we have no such certificate. */
+MOCK_IMPL(authority_cert_t *,
+get_my_v3_authority_cert, (void))
+{
+ return authority_key_certificate;
+}
+
+/** Return the v3 signing key for this v3 (voting) authority, or NULL
+ * if we have no such key. */
+crypto_pk_t *
+get_my_v3_authority_signing_key(void)
+{
+ return authority_signing_key;
+}
+
+/** If we're an authority, and we're using a legacy authority identity key for
+ * emergency migration purposes, return the certificate associated with that
+ * key. */
+authority_cert_t *
+get_my_v3_legacy_cert(void)
+{
+ return legacy_key_certificate;
+}
+
+/** If we're an authority, and we're using a legacy authority identity key for
+ * emergency migration purposes, return that key. */
+crypto_pk_t *
+get_my_v3_legacy_signing_key(void)
+{
+ return legacy_signing_key;
+}
+
+/** Replace the previous onion key with the current onion key, and generate
+ * a new previous onion key. Immediately after calling this function,
+ * the OR should:
+ * - schedule all previous cpuworkers to shut down _after_ processing
+ * pending work. (This will cause fresh cpuworkers to be generated.)
+ * - generate and upload a fresh routerinfo.
+ */
+void
+rotate_onion_key(void)
+{
+ char *fname, *fname_prev;
+ crypto_pk_t *prkey = NULL;
+ or_state_t *state = get_or_state();
+ curve25519_keypair_t new_curve25519_keypair;
+ time_t now;
+ fname = get_keydir_fname("secret_onion_key");
+ fname_prev = get_keydir_fname("secret_onion_key.old");
+ /* There isn't much point replacing an old key with an empty file */
+ if (file_status(fname) == FN_FILE) {
+ if (replace_file(fname, fname_prev))
+ goto error;
+ }
+ if (!(prkey = crypto_pk_new())) {
+ log_err(LD_GENERAL,"Error constructing rotated onion key");
+ goto error;
+ }
+ if (crypto_pk_generate_key(prkey)) {
+ log_err(LD_BUG,"Error generating onion key");
+ goto error;
+ }
+ if (crypto_pk_write_private_key_to_filename(prkey, fname)) {
+ log_err(LD_FS,"Couldn't write generated onion key to \"%s\".", fname);
+ goto error;
+ }
+ tor_free(fname);
+ tor_free(fname_prev);
+ fname = get_keydir_fname("secret_onion_key_ntor");
+ fname_prev = get_keydir_fname("secret_onion_key_ntor.old");
+ if (curve25519_keypair_generate(&new_curve25519_keypair, 1) < 0)
+ goto error;
+ /* There isn't much point replacing an old key with an empty file */
+ if (file_status(fname) == FN_FILE) {
+ if (replace_file(fname, fname_prev))
+ goto error;
+ }
+ if (curve25519_keypair_write_to_file(&new_curve25519_keypair, fname,
+ "onion") < 0) {
+ log_err(LD_FS,"Couldn't write curve25519 onion key to \"%s\".",fname);
+ goto error;
+ }
+ log_info(LD_GENERAL, "Rotating onion key");
+ tor_mutex_acquire(key_lock);
+ crypto_pk_free(lastonionkey);
+ lastonionkey = onionkey;
+ onionkey = prkey;
+ memcpy(&last_curve25519_onion_key, &curve25519_onion_key,
+ sizeof(curve25519_keypair_t));
+ memcpy(&curve25519_onion_key, &new_curve25519_keypair,
+ sizeof(curve25519_keypair_t));
+ now = time(NULL);
+ state->LastRotatedOnionKey = onionkey_set_at = now;
+ tor_mutex_release(key_lock);
+ mark_my_descriptor_dirty("rotated onion key");
+ or_state_mark_dirty(state, get_options()->AvoidDiskWrites ? now+3600 : 0);
+ goto done;
+ error:
+ log_warn(LD_GENERAL, "Couldn't rotate onion key.");
+ if (prkey)
+ crypto_pk_free(prkey);
+ done:
+ memwipe(&new_curve25519_keypair, 0, sizeof(new_curve25519_keypair));
+ tor_free(fname);
+ tor_free(fname_prev);
+}
+
+/** Log greeting message that points to new relay lifecycle document the
+ * first time this function has been called.
+ */
+static void
+log_new_relay_greeting(void)
+{
+ static int already_logged = 0;
+
+ if (already_logged)
+ return;
+
+ tor_log(LOG_NOTICE, LD_GENERAL, "You are running a new relay. "
+ "Thanks for helping the Tor network! If you wish to know "
+ "what will happen in the upcoming weeks regarding its usage, "
+ "have a look at https://blog.torproject.org/blog/lifecycle-of";
+ "-a-new-relay");
+
+ already_logged = 1;
+}
+
+/** Try to read an RSA key from <b>fname</b>. If <b>fname</b> doesn't exist
+ * and <b>generate</b> is true, create a new RSA key and save it in
+ * <b>fname</b>. Return the read/created key, or NULL on error. Log all
+ * errors at level <b>severity</b>. If <b>log_greeting</b> is non-zero and a
+ * new key was created, log_new_relay_greeting() is called.
+ */
+crypto_pk_t *
+init_key_from_file(const char *fname, int generate, int severity,
+ int log_greeting)
+{
+ crypto_pk_t *prkey = NULL;
+
+ if (!(prkey = crypto_pk_new())) {
+ tor_log(severity, LD_GENERAL,"Error constructing key");
+ goto error;
+ }
+
+ switch (file_status(fname)) {
+ case FN_DIR:
+ case FN_ERROR:
+ tor_log(severity, LD_FS,"Can't read key from \"%s\"", fname);
+ goto error;
+ /* treat empty key files as if the file doesn't exist, and,
+ * if generate is set, replace the empty file in
+ * crypto_pk_write_private_key_to_filename() */
+ case FN_NOENT:
+ case FN_EMPTY:
+ if (generate) {
+ if (!have_lockfile()) {
+ if (try_locking(get_options(), 0)<0) {
+ /* Make sure that --list-fingerprint only creates new keys
+ * if there is no possibility for a deadlock. */
+ tor_log(severity, LD_FS, "Another Tor process has locked \"%s\". "
+ "Not writing any new keys.", fname);
+ /*XXXX The 'other process' might make a key in a second or two;
+ * maybe we should wait for it. */
+ goto error;
+ }
+ }
+ log_info(LD_GENERAL, "No key found in \"%s\"; generating fresh key.",
+ fname);
+ if (crypto_pk_generate_key(prkey)) {
+ tor_log(severity, LD_GENERAL,"Error generating onion key");
+ goto error;
+ }
+ if (crypto_pk_check_key(prkey) <= 0) {
+ tor_log(severity, LD_GENERAL,"Generated key seems invalid");
+ goto error;
+ }
+ log_info(LD_GENERAL, "Generated key seems valid");
+ if (log_greeting) {
+ log_new_relay_greeting();
+ }
+ if (crypto_pk_write_private_key_to_filename(prkey, fname)) {
+ tor_log(severity, LD_FS,
+ "Couldn't write generated key to \"%s\".", fname);
+ goto error;
+ }
+ } else {
+ tor_log(severity, LD_GENERAL, "No key found in \"%s\"", fname);
+ goto error;
+ }
+ return prkey;
+ case FN_FILE:
+ if (crypto_pk_read_private_key_from_filename(prkey, fname)) {
+ tor_log(severity, LD_GENERAL,"Error loading private key.");
+ goto error;
+ }
+ return prkey;
+ default:
+ tor_assert(0);
+ }
+
+ error:
+ if (prkey)
+ crypto_pk_free(prkey);
+ return NULL;
+}
+
+/** Load a curve25519 keypair from the file <b>fname</b>, writing it into
+ * <b>keys_out</b>. If the file isn't found, or is empty, and <b>generate</b>
+ * is true, create a new keypair and write it into the file. If there are
+ * errors, log them at level <b>severity</b>. Generate files using <b>tag</b>
+ * in their ASCII wrapper. */
+static int
+init_curve25519_keypair_from_file(curve25519_keypair_t *keys_out,
+ const char *fname,
+ int generate,
+ int severity,
+ const char *tag)
+{
+ switch (file_status(fname)) {
+ case FN_DIR:
+ case FN_ERROR:
+ tor_log(severity, LD_FS,"Can't read key from \"%s\"", fname);
+ goto error;
+ /* treat empty key files as if the file doesn't exist, and, if generate
+ * is set, replace the empty file in curve25519_keypair_write_to_file() */
+ case FN_NOENT:
+ case FN_EMPTY:
+ if (generate) {
+ if (!have_lockfile()) {
+ if (try_locking(get_options(), 0)<0) {
+ /* Make sure that --list-fingerprint only creates new keys
+ * if there is no possibility for a deadlock. */
+ tor_log(severity, LD_FS, "Another Tor process has locked \"%s\". "
+ "Not writing any new keys.", fname);
+ /*XXXX The 'other process' might make a key in a second or two;
+ * maybe we should wait for it. */
+ goto error;
+ }
+ }
+ log_info(LD_GENERAL, "No key found in \"%s\"; generating fresh key.",
+ fname);
+ if (curve25519_keypair_generate(keys_out, 1) < 0)
+ goto error;
+ if (curve25519_keypair_write_to_file(keys_out, fname, tag)<0) {
+ tor_log(severity, LD_FS,
+ "Couldn't write generated key to \"%s\".", fname);
+ memwipe(keys_out, 0, sizeof(*keys_out));
+ goto error;
+ }
+ } else {
+ log_info(LD_GENERAL, "No key found in \"%s\"", fname);
+ }
+ return 0;
+ case FN_FILE:
+ {
+ char *tag_in=NULL;
+ if (curve25519_keypair_read_from_file(keys_out, &tag_in, fname) < 0) {
+ tor_log(severity, LD_GENERAL,"Error loading private key.");
+ tor_free(tag_in);
+ goto error;
+ }
+ if (!tag_in || strcmp(tag_in, tag)) {
+ tor_log(severity, LD_GENERAL,"Unexpected tag %s on private key.",
+ escaped(tag_in));
+ tor_free(tag_in);
+ goto error;
+ }
+ tor_free(tag_in);
+ return 0;
+ }
+ default:
+ tor_assert(0);
+ }
+
+ error:
+ return -1;
+}
+
+/** Try to load the vote-signing private key and certificate for being a v3
+ * directory authority, and make sure they match. If <b>legacy</b>, load a
+ * legacy key/cert set for emergency key migration; otherwise load the regular
+ * key/cert set. On success, store them into *<b>key_out</b> and
+ * *<b>cert_out</b> respectively, and return 0. On failure, return -1. */
+static int
+load_authority_keyset(int legacy, crypto_pk_t **key_out,
+ authority_cert_t **cert_out)
+{
+ int r = -1;
+ char *fname = NULL, *cert = NULL;
+ const char *eos = NULL;
+ crypto_pk_t *signing_key = NULL;
+ authority_cert_t *parsed = NULL;
+
+ fname = get_keydir_fname(
+ legacy ? "legacy_signing_key" : "authority_signing_key");
+ signing_key = init_key_from_file(fname, 0, LOG_ERR, 0);
+ if (!signing_key) {
+ log_warn(LD_DIR, "No version 3 directory key found in %s", fname);
+ goto done;
+ }
+ tor_free(fname);
+ fname = get_keydir_fname(
+ legacy ? "legacy_certificate" : "authority_certificate");
+ cert = read_file_to_str(fname, 0, NULL);
+ if (!cert) {
+ log_warn(LD_DIR, "Signing key found, but no certificate found in %s",
+ fname);
+ goto done;
+ }
+ parsed = authority_cert_parse_from_string(cert, &eos);
+ if (!parsed) {
+ log_warn(LD_DIR, "Unable to parse certificate in %s", fname);
+ goto done;
+ }
+ if (!crypto_pk_eq_keys(signing_key, parsed->signing_key)) {
+ log_warn(LD_DIR, "Stored signing key does not match signing key in "
+ "certificate");
+ goto done;
+ }
+
+ crypto_pk_free(*key_out);
+ authority_cert_free(*cert_out);
+
+ *key_out = signing_key;
+ *cert_out = parsed;
+ r = 0;
+ signing_key = NULL;
+ parsed = NULL;
+
+ done:
+ tor_free(fname);
+ tor_free(cert);
+ crypto_pk_free(signing_key);
+ authority_cert_free(parsed);
+ return r;
+}
+
+/** Load the v3 (voting) authority signing key and certificate, if they are
+ * present. Return -1 if anything is missing, mismatched, or unloadable;
+ * return 0 on success. */
+static int
+init_v3_authority_keys(void)
+{
+ if (load_authority_keyset(0, &authority_signing_key,
+ &authority_key_certificate)<0)
+ return -1;
+
+ if (get_options()->V3AuthUseLegacyKey &&
+ load_authority_keyset(1, &legacy_signing_key,
+ &legacy_key_certificate)<0)
+ return -1;
+
+ return 0;
+}
+
+/** If we're a v3 authority, check whether we have a certificate that's
+ * likely to expire soon. Warn if we do, but not too often. */
+void
+v3_authority_check_key_expiry(void)
+{
+ time_t now, expires;
+ static time_t last_warned = 0;
+ int badness, time_left, warn_interval;
+ if (!authdir_mode_v3(get_options()) || !authority_key_certificate)
+ return;
+
+ now = time(NULL);
+ expires = authority_key_certificate->expires;
+ time_left = (int)( expires - now );
+ if (time_left <= 0) {
+ badness = LOG_ERR;
+ warn_interval = 60*60;
+ } else if (time_left <= 24*60*60) {
+ badness = LOG_WARN;
+ warn_interval = 60*60;
+ } else if (time_left <= 24*60*60*7) {
+ badness = LOG_WARN;
+ warn_interval = 24*60*60;
+ } else if (time_left <= 24*60*60*30) {
+ badness = LOG_WARN;
+ warn_interval = 24*60*60*5;
+ } else {
+ return;
+ }
+
+ if (last_warned + warn_interval > now)
+ return;
+
+ if (time_left <= 0) {
+ tor_log(badness, LD_DIR, "Your v3 authority certificate has expired."
+ " Generate a new one NOW.");
+ } else if (time_left <= 24*60*60) {
+ tor_log(badness, LD_DIR, "Your v3 authority certificate expires in %d "
+ "hours; Generate a new one NOW.", time_left/(60*60));
+ } else {
+ tor_log(badness, LD_DIR, "Your v3 authority certificate expires in %d "
+ "days; Generate a new one soon.", time_left/(24*60*60));
+ }
+ last_warned = now;
+}
+
+/** Get the lifetime of an onion key in days. This value is defined by the
+ * network consesus parameter "onion-key-rotation-days". Always returns a value
+ * between <b>MIN_ONION_KEY_LIFETIME_DAYS</b> and
+ * <b>MAX_ONION_KEY_LIFETIME_DAYS</b>.
+ */
+static int
+get_onion_key_rotation_days_(void)
+{
+ return networkstatus_get_param(NULL,
+ "onion-key-rotation-days",
+ DEFAULT_ONION_KEY_LIFETIME_DAYS,
+ MIN_ONION_KEY_LIFETIME_DAYS,
+ MAX_ONION_KEY_LIFETIME_DAYS);
+}
+
+/** Get the current lifetime of an onion key in seconds. This value is defined
+ * by the network consesus parameter "onion-key-rotation-days", but the value
+ * is converted to seconds.
+ */
+int
+get_onion_key_lifetime(void)
+{
+ return get_onion_key_rotation_days_()*24*60*60;
+}
+
+/** Get the grace period of an onion key in seconds. This value is defined by
+ * the network consesus parameter "onion-key-grace-period-days", but the value
+ * is converted to seconds.
+ */
+int
+get_onion_key_grace_period(void)
+{
+ int grace_period;
+ grace_period = networkstatus_get_param(NULL,
+ "onion-key-grace-period-days",
+ DEFAULT_ONION_KEY_GRACE_PERIOD_DAYS,
+ MIN_ONION_KEY_GRACE_PERIOD_DAYS,
+ get_onion_key_rotation_days_());
+ return grace_period*24*60*60;
+}
+
+/** Set up Tor's TLS contexts, based on our configuration and keys. Return 0
+ * on success, and -1 on failure. */
+int
+router_initialize_tls_context(void)
+{
+ unsigned int flags = 0;
+ const or_options_t *options = get_options();
+ int lifetime = options->SSLKeyLifetime;
+ if (public_server_mode(options))
+ flags |= TOR_TLS_CTX_IS_PUBLIC_SERVER;
+ if (!lifetime) { /* we should guess a good ssl cert lifetime */
+
+ /* choose between 5 and 365 days, and round to the day */
+ unsigned int five_days = 5*24*3600;
+ unsigned int one_year = 365*24*3600;
+ lifetime = crypto_rand_int_range(five_days, one_year);
+ lifetime -= lifetime % (24*3600);
+
+ if (crypto_rand_int(2)) {
+ /* Half the time we expire at midnight, and half the time we expire
+ * one second before midnight. (Some CAs wobble their expiry times a
+ * bit in practice, perhaps to reduce collision attacks; see ticket
+ * 8443 for details about observed certs in the wild.) */
+ lifetime--;
+ }
+ }
+
+ /* It's ok to pass lifetime in as an unsigned int, since
+ * config_parse_interval() checked it. */
+ return tor_tls_context_init(flags,
+ get_tlsclient_identity_key(),
+ server_mode(options) ?
+ get_server_identity_key() : NULL,
+ (unsigned int)lifetime);
+}
+
+/** Compute fingerprint (or hashed fingerprint if hashed is 1) and write
+ * it to 'fingerprint' (or 'hashed-fingerprint'). Return 0 on success, or
+ * -1 if Tor should die,
+ */
+STATIC int
+router_write_fingerprint(int hashed)
+{
+ char *keydir = NULL, *cp = NULL;
+ const char *fname = hashed ? "hashed-fingerprint" :
+ "fingerprint";
+ char fingerprint[FINGERPRINT_LEN+1];
+ const or_options_t *options = get_options();
+ char *fingerprint_line = NULL;
+ int result = -1;
+
+ keydir = get_datadir_fname(fname);
+ log_info(LD_GENERAL,"Dumping %sfingerprint to \"%s\"...",
+ hashed ? "hashed " : "", keydir);
+ if (!hashed) {
+ if (crypto_pk_get_fingerprint(get_server_identity_key(),
+ fingerprint, 0) < 0) {
+ log_err(LD_GENERAL,"Error computing fingerprint");
+ goto done;
+ }
+ } else {
+ if (crypto_pk_get_hashed_fingerprint(get_server_identity_key(),
+ fingerprint) < 0) {
+ log_err(LD_GENERAL,"Error computing hashed fingerprint");
+ goto done;
+ }
+ }
+
+ tor_asprintf(&fingerprint_line, "%s %s\n", options->Nickname, fingerprint);
+
+ /* Check whether we need to write the (hashed-)fingerprint file. */
+
+ cp = read_file_to_str(keydir, RFTS_IGNORE_MISSING, NULL);
+ if (!cp || strcmp(cp, fingerprint_line)) {
+ if (write_str_to_file(keydir, fingerprint_line, 0)) {
+ log_err(LD_FS, "Error writing %sfingerprint line to file",
+ hashed ? "hashed " : "");
+ goto done;
+ }
+ }
+
+ log_notice(LD_GENERAL, "Your Tor %s identity key fingerprint is '%s %s'",
+ hashed ? "bridge's hashed" : "server's", options->Nickname,
+ fingerprint);
+
+ result = 0;
+ done:
+ tor_free(cp);
+ tor_free(keydir);
+ tor_free(fingerprint_line);
+ return result;
+}
+
+static int
+init_keys_common(void)
+{
+ if (!key_lock)
+ key_lock = tor_mutex_new();
+
+ /* There are a couple of paths that put us here before we've asked
+ * openssl to initialize itself. */
+ if (crypto_global_init(get_options()->HardwareAccel,
+ get_options()->AccelName,
+ get_options()->AccelDir)) {
+ log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting.");
+ return -1;
+ }
+
+ return 0;
+}
+
+int
+init_keys_client(void)
+{
+ crypto_pk_t *prkey;
+ if (init_keys_common() < 0)
+ return -1;
+
+ if (!(prkey = crypto_pk_new()))
+ return -1;
+ if (crypto_pk_generate_key(prkey)) {
+ crypto_pk_free(prkey);
+ return -1;
+ }
+ set_client_identity_key(prkey);
+ /* Create a TLS context. */
+ if (router_initialize_tls_context() < 0) {
+ log_err(LD_GENERAL,"Error creating TLS context for Tor client.");
+ return -1;
+ }
+ return 0;
+}
+
+/** Initialize all OR private keys, and the TLS context, as necessary.
+ * On OPs, this only initializes the tls context. Return 0 on success,
+ * or -1 if Tor should die.
+ */
+int
+init_keys(void)
+{
+ char *keydir;
+ const char *mydesc;
+ crypto_pk_t *prkey;
+ char digest[DIGEST_LEN];
+ char v3_digest[DIGEST_LEN];
+ const or_options_t *options = get_options();
+ dirinfo_type_t type;
+ time_t now = time(NULL);
+ dir_server_t *ds;
+ int v3_digest_set = 0;
+ authority_cert_t *cert = NULL;
+
+ /* OP's don't need persistent keys; just make up an identity and
+ * initialize the TLS context. */
+ if (!server_mode(options)) {
+ return init_keys_client();
+ }
+ if (init_keys_common() < 0)
+ return -1;
+
+ if (create_keys_directory(options) < 0)
+ return -1;
+
+ /* 1a. Read v3 directory authority key/cert information. */
+ memset(v3_digest, 0, sizeof(v3_digest));
+ if (authdir_mode_v3(options)) {
+ if (init_v3_authority_keys()<0) {
+ log_err(LD_GENERAL, "We're configured as a V3 authority, but we "
+ "were unable to load our v3 authority keys and certificate! "
+ "Use tor-gencert to generate them. Dying.");
+ return -1;
+ }
+ cert = get_my_v3_authority_cert();
+ if (cert) {
+ if (crypto_pk_get_digest(get_my_v3_authority_cert()->identity_key,
+ v3_digest) < 0) {
+ log_err(LD_BUG, "Couldn't compute my v3 authority identity key "
+ "digest.");
+ return -1;
+ }
+ v3_digest_set = 1;
+ }
+ }
+
+ /* 1b. Read identity key. Make it if none is found. */
+ keydir = get_keydir_fname("secret_id_key");
+ log_info(LD_GENERAL,"Reading/making identity key \"%s\"...",keydir);
+ prkey = init_key_from_file(keydir, 1, LOG_ERR, 1);
+ tor_free(keydir);
+ if (!prkey) return -1;
+ set_server_identity_key(prkey);
+
+ /* 1c. If we are configured as a bridge, generate a client key;
+ * otherwise, set the server identity key as our client identity
+ * key. */
+ if (public_server_mode(options)) {
+ set_client_identity_key(crypto_pk_dup_key(prkey)); /* set above */
+ } else {
+ if (!(prkey = crypto_pk_new()))
+ return -1;
+ if (crypto_pk_generate_key(prkey)) {
+ crypto_pk_free(prkey);
+ return -1;
+ }
+ set_client_identity_key(prkey);
+ }
+
+ /* 1d. Load all ed25519 keys */
+ const int new_signing_key = load_ed_keys(options,now);
+ if (new_signing_key < 0)
+ return -1;
+
+ /* 2. Read onion key. Make it if none is found. */
+ keydir = get_keydir_fname("secret_onion_key");
+ log_info(LD_GENERAL,"Reading/making onion key \"%s\"...",keydir);
+ prkey = init_key_from_file(keydir, 1, LOG_ERR, 1);
+ tor_free(keydir);
+ if (!prkey) return -1;
+ set_onion_key(prkey);
+ if (options->command == CMD_RUN_TOR) {
+ /* only mess with the state file if we're actually running Tor */
+ or_state_t *state = get_or_state();
+ if (state->LastRotatedOnionKey > 100 && state->LastRotatedOnionKey < now) {
+ /* We allow for some parsing slop, but we don't want to risk accepting
+ * values in the distant future. If we did, we might never rotate the
+ * onion key. */
+ onionkey_set_at = state->LastRotatedOnionKey;
+ } else {
+ /* We have no LastRotatedOnionKey set; either we just created the key
+ * or it's a holdover from 0.1.2.4-alpha-dev or earlier. In either case,
+ * start the clock ticking now so that we will eventually rotate it even
+ * if we don't stay up for the full lifetime of an onion key. */
+ state->LastRotatedOnionKey = onionkey_set_at = now;
+ or_state_mark_dirty(state, options->AvoidDiskWrites ?
+ time(NULL)+3600 : 0);
+ }
+ }
+
+ keydir = get_keydir_fname("secret_onion_key.old");
+ if (!lastonionkey && file_status(keydir) == FN_FILE) {
+ /* Load keys from non-empty files only.
+ * Missing old keys won't be replaced with freshly generated keys. */
+ prkey = init_key_from_file(keydir, 0, LOG_ERR, 0);
+ if (prkey)
+ lastonionkey = prkey;
+ }
+ tor_free(keydir);
+
+ {
+ /* 2b. Load curve25519 onion keys. */
+ int r;
+ keydir = get_keydir_fname("secret_onion_key_ntor");
+ r = init_curve25519_keypair_from_file(&curve25519_onion_key,
+ keydir, 1, LOG_ERR, "onion");
+ tor_free(keydir);
+ if (r<0)
+ return -1;
+
+ keydir = get_keydir_fname("secret_onion_key_ntor.old");
+ if (tor_mem_is_zero((const char *)
+ last_curve25519_onion_key.pubkey.public_key,
+ CURVE25519_PUBKEY_LEN) &&
+ file_status(keydir) == FN_FILE) {
+ /* Load keys from non-empty files only.
+ * Missing old keys won't be replaced with freshly generated keys. */
+ init_curve25519_keypair_from_file(&last_curve25519_onion_key,
+ keydir, 0, LOG_ERR, "onion");
+ }
+ tor_free(keydir);
+ }
+
+ /* 3. Initialize link key and TLS context. */
+ if (router_initialize_tls_context() < 0) {
+ log_err(LD_GENERAL,"Error initializing TLS context");
+ return -1;
+ }
+
+ /* 3b. Get an ed25519 link certificate. Note that we need to do this
+ * after we set up the TLS context */
+ if (generate_ed_link_cert(options, now, new_signing_key > 0) < 0) {
+ log_err(LD_GENERAL,"Couldn't make link cert");
+ return -1;
+ }
+
+ /* 4. Build our router descriptor. */
+ /* Must be called after keys are initialized. */
+ mydesc = router_get_my_descriptor();
+ if (authdir_mode_v3(options)) {
+ const char *m = NULL;
+ routerinfo_t *ri;
+ /* We need to add our own fingerprint so it gets recognized. */
+ if (dirserv_add_own_fingerprint(get_server_identity_key())) {
+ log_err(LD_GENERAL,"Error adding own fingerprint to set of relays");
+ return -1;
+ }
+ if (mydesc) {
+ was_router_added_t added;
+ ri = router_parse_entry_from_string(mydesc, NULL, 1, 0, NULL, NULL);
+ if (!ri) {
+ log_err(LD_GENERAL,"Generated a routerinfo we couldn't parse.");
+ return -1;
+ }
+ added = dirserv_add_descriptor(ri, &m, "self");
+ if (!WRA_WAS_ADDED(added)) {
+ if (!WRA_WAS_OUTDATED(added)) {
+ log_err(LD_GENERAL, "Unable to add own descriptor to directory: %s",
+ m?m:"<unknown error>");
+ return -1;
+ } else {
+ /* If the descriptor was outdated, that's ok. This can happen
+ * when some config options are toggled that affect workers, but
+ * we don't really need new keys yet so the descriptor doesn't
+ * change and the old one is still fresh. */
+ log_info(LD_GENERAL, "Couldn't add own descriptor to directory "
+ "after key init: %s This is usually not a problem.",
+ m?m:"<unknown error>");
+ }
+ }
+ }
+ }
+
+ /* 5. Dump fingerprint and possibly hashed fingerprint to files. */
+ if (router_write_fingerprint(0)) {
+ log_err(LD_FS, "Error writing fingerprint to file");
+ return -1;
+ }
+ if (!public_server_mode(options) && router_write_fingerprint(1)) {
+ log_err(LD_FS, "Error writing hashed fingerprint to file");
+ return -1;
+ }
+
+ if (!authdir_mode(options))
+ return 0;
+ /* 6. [authdirserver only] load approved-routers file */
+ if (dirserv_load_fingerprint_file() < 0) {
+ log_err(LD_GENERAL,"Error loading fingerprints");
+ return -1;
+ }
+ /* 6b. [authdirserver only] add own key to approved directories. */
+ crypto_pk_get_digest(get_server_identity_key(), digest);
+ type = ((options->V3AuthoritativeDir ?
+ (V3_DIRINFO|MICRODESC_DIRINFO|EXTRAINFO_DIRINFO) : NO_DIRINFO) |
+ (options->BridgeAuthoritativeDir ? BRIDGE_DIRINFO : NO_DIRINFO));
+
+ ds = router_get_trusteddirserver_by_digest(digest);
+ if (!ds) {
+ ds = trusted_dir_server_new(options->Nickname, NULL,
+ router_get_advertised_dir_port(options, 0),
+ router_get_advertised_or_port(options),
+ NULL,
+ digest,
+ v3_digest,
+ type, 0.0);
+ if (!ds) {
+ log_err(LD_GENERAL,"We want to be a directory authority, but we "
+ "couldn't add ourselves to the authority list. Failing.");
+ return -1;
+ }
+ dir_server_add(ds);
+ }
+ if (ds->type != type) {
+ log_warn(LD_DIR, "Configured authority type does not match authority "
+ "type in DirAuthority list. Adjusting. (%d v %d)",
+ type, ds->type);
+ ds->type = type;
+ }
+ if (v3_digest_set && (ds->type & V3_DIRINFO) &&
+ tor_memneq(v3_digest, ds->v3_identity_digest, DIGEST_LEN)) {
+ log_warn(LD_DIR, "V3 identity key does not match identity declared in "
+ "DirAuthority line. Adjusting.");
+ memcpy(ds->v3_identity_digest, v3_digest, DIGEST_LEN);
+ }
+
+ if (cert) { /* add my own cert to the list of known certs */
+ log_info(LD_DIR, "adding my own v3 cert");
+ if (trusted_dirs_load_certs_from_string(
+ cert->cache_info.signed_descriptor_body,
+ TRUSTED_DIRS_CERTS_SRC_SELF, 0,
+ NULL)<0) {
+ log_warn(LD_DIR, "Unable to parse my own v3 cert! Failing.");
+ return -1;
+ }
+ }
+
+ return 0; /* success */
+}
+
+/* Keep track of whether we should upload our server descriptor,
+ * and what type of server we are.
+ */
+
+/** Whether we can reach our ORPort from the outside. */
+static int can_reach_or_port = 0;
+/** Whether we can reach our DirPort from the outside. */
+static int can_reach_dir_port = 0;
+
+/** Forget what we have learned about our reachability status. */
+void
+router_reset_reachability(void)
+{
+ can_reach_or_port = can_reach_dir_port = 0;
+}
+
+/** Return 1 if we won't do reachability checks, because:
+ * - AssumeReachable is set, or
+ * - the network is disabled.
+ * Otherwise, return 0.
+ */
+static int
+router_reachability_checks_disabled(const or_options_t *options)
+{
+ return options->AssumeReachable ||
+ net_is_disabled();
+}
+
+/** Return 0 if we need to do an ORPort reachability check, because:
+ * - no reachability check has been done yet, or
+ * - we've initiated reachability checks, but none have succeeded.
+ * Return 1 if we don't need to do an ORPort reachability check, because:
+ * - we've seen a successful reachability check, or
+ * - AssumeReachable is set, or
+ * - the network is disabled.
+ */
+int
+check_whether_orport_reachable(const or_options_t *options)
+{
+ int reach_checks_disabled = router_reachability_checks_disabled(options);
+ return reach_checks_disabled ||
+ can_reach_or_port;
+}
+
+/** Return 0 if we need to do a DirPort reachability check, because:
+ * - no reachability check has been done yet, or
+ * - we've initiated reachability checks, but none have succeeded.
+ * Return 1 if we don't need to do a DirPort reachability check, because:
+ * - we've seen a successful reachability check, or
+ * - there is no DirPort set, or
+ * - AssumeReachable is set, or
+ * - the network is disabled.
+ */
+int
+check_whether_dirport_reachable(const or_options_t *options)
+{
+ int reach_checks_disabled = router_reachability_checks_disabled(options) ||
+ !options->DirPort_set;
+ return reach_checks_disabled ||
+ can_reach_dir_port;
+}
+
+/** The lower threshold of remaining bandwidth required to advertise (or
+ * automatically provide) directory services */
+/* XXX Should this be increased? */
+#define MIN_BW_TO_ADVERTISE_DIRSERVER 51200
+
+/** Return true iff we have enough configured bandwidth to advertise or
+ * automatically provide directory services from cache directory
+ * information. */
+static int
+router_has_bandwidth_to_be_dirserver(const or_options_t *options)
+{
+ if (options->BandwidthRate < MIN_BW_TO_ADVERTISE_DIRSERVER) {
+ return 0;
+ }
+ if (options->RelayBandwidthRate > 0 &&
+ options->RelayBandwidthRate < MIN_BW_TO_ADVERTISE_DIRSERVER) {
+ return 0;
+ }
+ return 1;
+}
+
+/** Helper: Return 1 if we have sufficient resources for serving directory
+ * requests, return 0 otherwise.
+ * dir_port is either 0 or the configured DirPort number.
+ * If AccountingMax is set less than our advertised bandwidth, then don't
+ * serve requests. Likewise, if our advertised bandwidth is less than
+ * MIN_BW_TO_ADVERTISE_DIRSERVER, don't bother trying to serve requests.
+ */
+static int
+router_should_be_dirserver(const or_options_t *options, int dir_port)
+{
+ static int advertising=1; /* start out assuming we will advertise */
+ int new_choice=1;
+ const char *reason = NULL;
+
+ if (accounting_is_enabled(options) &&
+ get_options()->AccountingRule != ACCT_IN) {
+ /* Don't spend bytes for directory traffic if we could end up hibernating,
+ * but allow DirPort otherwise. Some relay operators set AccountingMax
+ * because they're confused or to get statistics. Directory traffic has a
+ * much larger effect on output than input so there is no reason to turn it
+ * off if using AccountingRule in. */
+ int interval_length = accounting_get_interval_length();
+ uint32_t effective_bw = get_effective_bwrate(options);
+ uint64_t acc_bytes;
+ if (!interval_length) {
+ log_warn(LD_BUG, "An accounting interval is not allowed to be zero "
+ "seconds long. Raising to 1.");
+ interval_length = 1;
+ }
+ log_info(LD_GENERAL, "Calculating whether to advertise %s: effective "
+ "bwrate: %u, AccountingMax: %"PRIu64", "
+ "accounting interval length %d",
+ dir_port ? "dirport" : "begindir",
+ effective_bw, (options->AccountingMax),
+ interval_length);
+
+ acc_bytes = options->AccountingMax;
+ if (get_options()->AccountingRule == ACCT_SUM)
+ acc_bytes /= 2;
+ if (effective_bw >=
+ acc_bytes / interval_length) {
+ new_choice = 0;
+ reason = "AccountingMax enabled";
+ }
+ } else if (! router_has_bandwidth_to_be_dirserver(options)) {
+ /* if we're advertising a small amount */
+ new_choice = 0;
+ reason = "BandwidthRate under 50KB";
+ }
+
+ if (advertising != new_choice) {
+ if (new_choice == 1) {
+ if (dir_port > 0)
+ log_notice(LD_DIR, "Advertising DirPort as %d", dir_port);
+ else
+ log_notice(LD_DIR, "Advertising directory service support");
+ } else {
+ tor_assert(reason);
+ log_notice(LD_DIR, "Not advertising Dir%s (Reason: %s)",
+ dir_port ? "Port" : "ectory Service support", reason);
+ }
+ advertising = new_choice;
+ }
+
+ return advertising;
+}
+
+/** Return 1 if we are configured to accept either relay or directory requests
+ * from clients and we aren't at risk of exceeding our bandwidth limits, thus
+ * we should be a directory server. If not, return 0.
+ */
+int
+dir_server_mode(const or_options_t *options)
+{
+ if (!options->DirCache)
+ return 0;
+ return options->DirPort_set ||
+ (server_mode(options) && router_has_bandwidth_to_be_dirserver(options));
+}
+
+/** Look at a variety of factors, and return 0 if we don't want to
+ * advertise the fact that we have a DirPort open or begindir support, else
+ * return 1.
+ *
+ * Where dir_port or supports_tunnelled_dir_requests are not relevant, they
+ * must be 0.
+ *
+ * Log a helpful message if we change our mind about whether to publish.
+ */
+static int
+decide_to_advertise_dir_impl(const or_options_t *options,
+ uint16_t dir_port,
+ int supports_tunnelled_dir_requests)
+{
+ /* Part one: reasons to publish or not publish that aren't
+ * worth mentioning to the user, either because they're obvious
+ * or because they're normal behavior. */
+
+ /* short circuit the rest of the function */
+ if (!dir_port && !supports_tunnelled_dir_requests)
+ return 0;
+ if (authdir_mode(options)) /* always publish */
+ return 1;
+ if (net_is_disabled())
+ return 0;
+ if (dir_port && !router_get_advertised_dir_port(options, dir_port))
+ return 0;
+ if (supports_tunnelled_dir_requests &&
+ !router_get_advertised_or_port(options))
+ return 0;
+
+ /* Part two: consider config options that could make us choose to
+ * publish or not publish that the user might find surprising. */
+ return router_should_be_dirserver(options, dir_port);
+}
+
+/** Front-end to decide_to_advertise_dir_impl(): return 0 if we don't want to
+ * advertise the fact that we have a DirPort open, else return the
+ * DirPort we want to advertise.
+ */
+static int
+router_should_advertise_dirport(const or_options_t *options, uint16_t dir_port)
+{
+ /* supports_tunnelled_dir_requests is not relevant, pass 0 */
+ return decide_to_advertise_dir_impl(options, dir_port, 0) ? dir_port : 0;
+}
+
+/** Front-end to decide_to_advertise_dir_impl(): return 0 if we don't want to
+ * advertise the fact that we support begindir requests, else return 1.
+ */
+static int
+router_should_advertise_begindir(const or_options_t *options,
+ int supports_tunnelled_dir_requests)
+{
+ /* dir_port is not relevant, pass 0 */
+ return decide_to_advertise_dir_impl(options, 0,
+ supports_tunnelled_dir_requests);
+}
+
+/** Allocate and return a new extend_info_t that can be used to build
+ * a circuit to or through the router <b>r</b>. Uses the primary
+ * address of the router, so should only be called on a server. */
+static extend_info_t *
+extend_info_from_router(const routerinfo_t *r)
+{
+ crypto_pk_t *rsa_pubkey;
+ extend_info_t *info;
+ tor_addr_port_t ap;
+ tor_assert(r);
+
+ /* Make sure we don't need to check address reachability */
+ tor_assert_nonfatal(router_skip_or_reachability(get_options(), 0));
+
+ const ed25519_public_key_t *ed_id_key;
+ if (r->cache_info.signing_key_cert)
+ ed_id_key = &r->cache_info.signing_key_cert->signing_key;
+ else
+ ed_id_key = NULL;
+
+ router_get_prim_orport(r, &ap);
+ rsa_pubkey = router_get_rsa_onion_pkey(r->onion_pkey, r->onion_pkey_len);
+ info = extend_info_new(r->nickname, r->cache_info.identity_digest,
+ ed_id_key,
+ rsa_pubkey, r->onion_curve25519_pkey,
+ &ap.addr, ap.port);
+ crypto_pk_free(rsa_pubkey);
+ return info;
+}
+
+/**See if we currently believe our ORPort or DirPort to be
+ * unreachable. If so, return 1 else return 0.
+ */
+static int
+router_should_check_reachability(int test_or, int test_dir)
+{
+ const routerinfo_t *me = router_get_my_routerinfo();
+ const or_options_t *options = get_options();
+
+ if (!me)
+ return 0;
+
+ if (routerset_contains_router(options->ExcludeNodes, me, -1) &&
+ options->StrictNodes) {
+ /* If we've excluded ourself, and StrictNodes is set, we can't test
+ * ourself. */
+ if (test_or || test_dir) {
+#define SELF_EXCLUDED_WARN_INTERVAL 3600
+ static ratelim_t warning_limit=RATELIM_INIT(SELF_EXCLUDED_WARN_INTERVAL);
+ log_fn_ratelim(&warning_limit, LOG_WARN, LD_CIRC,
+ "Can't peform self-tests for this relay: we have "
+ "listed ourself in ExcludeNodes, and StrictNodes is set. "
+ "We cannot learn whether we are usable, and will not "
+ "be able to advertise ourself.");
+ }
+ return 0;
+ }
+ return 1;
+}
+
+/** Some time has passed, or we just got new directory information.
+ * See if we currently believe our ORPort or DirPort to be
+ * unreachable. If so, launch a new test for it.
+ *
+ * For ORPort, we simply try making a circuit that ends at ourselves.
+ * Success is noticed in onionskin_answer().
+ *
+ * For DirPort, we make a connection via Tor to our DirPort and ask
+ * for our own server descriptor.
+ * Success is noticed in connection_dir_client_reached_eof().
+ */
+void
+router_do_reachability_checks(int test_or, int test_dir)
+{
+ const routerinfo_t *me = router_get_my_routerinfo();
+ const or_options_t *options = get_options();
+ int orport_reachable = check_whether_orport_reachable(options);
+ tor_addr_t addr;
+
+ if (router_should_check_reachability(test_or, test_dir)) {
+ if (test_or && (!orport_reachable || !circuit_enough_testing_circs())) {
+ extend_info_t *ei = extend_info_from_router(me);
+ /* XXX IPv6 self testing */
+ log_info(LD_CIRC, "Testing %s of my ORPort: %s:%d.",
+ !orport_reachable ? "reachability" : "bandwidth",
+ fmt_addr32(me->addr), me->or_port);
+ circuit_launch_by_extend_info(CIRCUIT_PURPOSE_TESTING, ei,
+ CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL);
+ extend_info_free(ei);
+ }
+
+ /* XXX IPv6 self testing */
+ tor_addr_from_ipv4h(&addr, me->addr);
+ if (test_dir && !check_whether_dirport_reachable(options) &&
+ !connection_get_by_type_addr_port_purpose(
+ CONN_TYPE_DIR, &addr, me->dir_port,
+ DIR_PURPOSE_FETCH_SERVERDESC)) {
+ tor_addr_port_t my_orport, my_dirport;
+ memcpy(&my_orport.addr, &addr, sizeof(addr));
+ memcpy(&my_dirport.addr, &addr, sizeof(addr));
+ my_orport.port = me->or_port;
+ my_dirport.port = me->dir_port;
+ /* ask myself, via tor, for my server descriptor. */
+ directory_request_t *req =
+ directory_request_new(DIR_PURPOSE_FETCH_SERVERDESC);
+ directory_request_set_or_addr_port(req, &my_orport);
+ directory_request_set_dir_addr_port(req, &my_dirport);
+ directory_request_set_directory_id_digest(req,
+ me->cache_info.identity_digest);
+ // ask via an anon circuit, connecting to our dirport.
+ directory_request_set_indirection(req, DIRIND_ANON_DIRPORT);
+ directory_request_set_resource(req, "authority.z");
+ directory_initiate_request(req);
+ directory_request_free(req);
+ }
+ }
+}
+
+/** Annotate that we found our ORPort reachable. */
+void
+router_orport_found_reachable(void)
+{
+ const routerinfo_t *me = router_get_my_routerinfo();
+ const or_options_t *options = get_options();
+ if (!can_reach_or_port && me) {
+ char *address = tor_dup_ip(me->addr);
+ log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from "
+ "the outside. Excellent.%s",
+ options->PublishServerDescriptor_ != NO_DIRINFO
+ && check_whether_dirport_reachable(options) ?
+ " Publishing server descriptor." : "");
+ can_reach_or_port = 1;
+ mark_my_descriptor_dirty("ORPort found reachable");
+ /* This is a significant enough change to upload immediately,
+ * at least in a test network */
+ if (options->TestingTorNetwork == 1) {
+ reschedule_descriptor_update_check();
+ }
+ control_event_server_status(LOG_NOTICE,
+ "REACHABILITY_SUCCEEDED ORADDRESS=%s:%d",
+ address, me->or_port);
+ tor_free(address);
+ }
+}
+
+/** Annotate that we found our DirPort reachable. */
+void
+router_dirport_found_reachable(void)
+{
+ const routerinfo_t *me = router_get_my_routerinfo();
+ const or_options_t *options = get_options();
+ if (!can_reach_dir_port && me) {
+ char *address = tor_dup_ip(me->addr);
+ log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable "
+ "from the outside. Excellent.%s",
+ options->PublishServerDescriptor_ != NO_DIRINFO
+ && check_whether_orport_reachable(options) ?
+ " Publishing server descriptor." : "");
+ can_reach_dir_port = 1;
+ if (router_should_advertise_dirport(options, me->dir_port)) {
+ mark_my_descriptor_dirty("DirPort found reachable");
+ /* This is a significant enough change to upload immediately,
+ * at least in a test network */
+ if (options->TestingTorNetwork == 1) {
+ reschedule_descriptor_update_check();
+ }
+ }
+ control_event_server_status(LOG_NOTICE,
+ "REACHABILITY_SUCCEEDED DIRADDRESS=%s:%d",
+ address, me->dir_port);
+ tor_free(address);
+ }
+}
+
+/** We have enough testing circuits open. Send a bunch of "drop"
+ * cells down each of them, to exercise our bandwidth. */
+void
+router_perform_bandwidth_test(int num_circs, time_t now)
+{
+ int num_cells = (int)(get_options()->BandwidthRate * 10 /
+ CELL_MAX_NETWORK_SIZE);
+ int max_cells = num_cells < CIRCWINDOW_START ?
+ num_cells : CIRCWINDOW_START;
+ int cells_per_circuit = max_cells / num_circs;
+ origin_circuit_t *circ = NULL;
+
+ log_notice(LD_OR,"Performing bandwidth self-test...done.");
+ while ((circ = circuit_get_next_by_pk_and_purpose(circ, NULL,
+ CIRCUIT_PURPOSE_TESTING))) {
+ /* dump cells_per_circuit drop cells onto this circ */
+ int i = cells_per_circuit;
+ if (circ->base_.state != CIRCUIT_STATE_OPEN)
+ continue;
+ circ->base_.timestamp_dirty = now;
+ while (i-- > 0) {
+ if (relay_send_command_from_edge(0, TO_CIRCUIT(circ),
+ RELAY_COMMAND_DROP,
+ NULL, 0, circ->cpath->prev)<0) {
+ return; /* stop if error */
+ }
+ }
+ }
+}
+
+/** Return true iff our network is in some sense disabled or shutting down:
+ * either we're hibernating, entering hibernation, or the network is turned
+ * off with DisableNetwork. */
+int
+net_is_disabled(void)
+{
+ return get_options()->DisableNetwork || we_are_hibernating();
+}
+
+/** Return true iff our network is in some sense "completely disabled" either
+ * we're fully hibernating or the network is turned off with
+ * DisableNetwork. */
+int
+net_is_completely_disabled(void)
+{
+ return get_options()->DisableNetwork || we_are_fully_hibernating();
+}
+
+/** Return true iff we believe ourselves to be an authoritative
+ * directory server.
+ */
+int
+authdir_mode(const or_options_t *options)
+{
+ return options->AuthoritativeDir != 0;
+}
+/** Return true iff we are an authoritative directory server that is
+ * authoritative about receiving and serving descriptors of type
+ * <b>purpose</b> on its dirport.
+ */
+int
+authdir_mode_handles_descs(const or_options_t *options, int purpose)
+{
+ if (BUG(purpose < 0)) /* Deprecated. */
+ return authdir_mode(options);
+ else if (purpose == ROUTER_PURPOSE_GENERAL)
+ return authdir_mode_v3(options);
+ else if (purpose == ROUTER_PURPOSE_BRIDGE)
+ return authdir_mode_bridge(options);
+ else
+ return 0;
+}
+/** Return true iff we are an authoritative directory server that
+ * publishes its own network statuses.
+ */
+int
+authdir_mode_publishes_statuses(const or_options_t *options)
+{
+ if (authdir_mode_bridge(options))
+ return 0;
+ return authdir_mode(options);
+}
+/** Return true iff we are an authoritative directory server that
+ * tests reachability of the descriptors it learns about.
+ */
+int
+authdir_mode_tests_reachability(const or_options_t *options)
+{
+ return authdir_mode(options);
+}
+/** Return true iff we believe ourselves to be a bridge authoritative
+ * directory server.
+ */
+int
+authdir_mode_bridge(const or_options_t *options)
+{
+ return authdir_mode(options) && options->BridgeAuthoritativeDir != 0;
+}
+
+/** Return true iff we are trying to be a server.
+ */
+MOCK_IMPL(int,
+server_mode,(const or_options_t *options))
+{
+ if (options->ClientOnly) return 0;
+ return (options->ORPort_set);
+}
+
+/** Return true iff we are trying to be a non-bridge server.
+ */
+MOCK_IMPL(int,
+public_server_mode,(const or_options_t *options))
+{
+ if (!server_mode(options)) return 0;
+ return (!options->BridgeRelay);
+}
+
+/** Return true iff the combination of options in <b>options</b> and parameters
+ * in the consensus mean that we don't want to allow exits from circuits
+ * we got from addresses not known to be servers. */
+int
+should_refuse_unknown_exits(const or_options_t *options)
+{
+ if (options->RefuseUnknownExits != -1) {
+ return options->RefuseUnknownExits;
+ } else {
+ return networkstatus_get_param(NULL, "refuseunknownexits", 1, 0, 1);
+ }
+}
+
+/** Remember if we've advertised ourselves to the dirservers. */
+static int server_is_advertised=0;
+
+/** Return true iff we have published our descriptor lately.
+ */
+MOCK_IMPL(int,
+advertised_server_mode,(void))
+{
+ return server_is_advertised;
+}
+
+/**
+ * Called with a boolean: set whether we have recently published our
+ * descriptor.
+ */
+static void
+set_server_advertised(int s)
+{
+ server_is_advertised = s;
+}
+
+/** Return true iff we are trying to proxy client connections. */
+int
+proxy_mode(const or_options_t *options)
+{
+ (void)options;
+ SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) {
+ if (p->type == CONN_TYPE_AP_LISTENER ||
+ p->type == CONN_TYPE_AP_TRANS_LISTENER ||
+ p->type == CONN_TYPE_AP_DNS_LISTENER ||
+ p->type == CONN_TYPE_AP_NATD_LISTENER)
+ return 1;
+ } SMARTLIST_FOREACH_END(p);
+ return 0;
+}
+
+/** Decide if we're a publishable server. We are a publishable server if:
+ * - We don't have the ClientOnly option set
+ * and
+ * - We have the PublishServerDescriptor option set to non-empty
+ * and
+ * - We have ORPort set
+ * and
+ * - We believe our ORPort and DirPort (if present) are reachable from
+ * the outside; or
+ * - We believe our ORPort is reachable from the outside, and we can't
+ * check our DirPort because the consensus has no exits; or
+ * - We are an authoritative directory server.
+ */
+static int
+decide_if_publishable_server(void)
+{
+ const or_options_t *options = get_options();
+
+ if (options->ClientOnly)
+ return 0;
+ if (options->PublishServerDescriptor_ == NO_DIRINFO)
+ return 0;
+ if (!server_mode(options))
+ return 0;
+ if (authdir_mode(options))
+ return 1;
+ if (!router_get_advertised_or_port(options))
+ return 0;
+ if (!check_whether_orport_reachable(options))
+ return 0;
+ if (router_have_consensus_path() == CONSENSUS_PATH_INTERNAL) {
+ /* All set: there are no exits in the consensus (maybe this is a tiny
+ * test network), so we can't check our DirPort reachability. */
+ return 1;
+ } else {
+ return check_whether_dirport_reachable(options);
+ }
+}
+
+/** Initiate server descriptor upload as reasonable (if server is publishable,
+ * etc). <b>force</b> is as for router_upload_dir_desc_to_dirservers.
+ *
+ * We need to rebuild the descriptor if it's dirty even if we're not
+ * uploading, because our reachability testing *uses* our descriptor to
+ * determine what IP address and ports to test.
+ */
+void
+consider_publishable_server(int force)
+{
+ int rebuilt;
+
+ if (!server_mode(get_options()))
+ return;
+
+ rebuilt = router_rebuild_descriptor(0);
+ if (decide_if_publishable_server()) {
+ set_server_advertised(1);
+ if (rebuilt == 0)
+ router_upload_dir_desc_to_dirservers(force);
+ } else {
+ set_server_advertised(0);
+ }
+}
+
+/** Return the port of the first active listener of type
+ * <b>listener_type</b>. */
+/** XXX not a very good interface. it's not reliable when there are
+ multiple listeners. */
+uint16_t
+router_get_active_listener_port_by_type_af(int listener_type,
+ sa_family_t family)
+{
+ /* Iterate all connections, find one of the right kind and return
+ the port. Not very sophisticated or fast, but effective. */
+ smartlist_t *conns = get_connection_array();
+ SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
+ if (conn->type == listener_type && !conn->marked_for_close &&
+ conn->socket_family == family) {
+ return conn->port;
+ }
+ } SMARTLIST_FOREACH_END(conn);
+
+ return 0;
+}
+
+/** Return the port that we should advertise as our ORPort; this is either
+ * the one configured in the ORPort option, or the one we actually bound to
+ * if ORPort is "auto".
+ */
+uint16_t
+router_get_advertised_or_port(const or_options_t *options)
+{
+ return router_get_advertised_or_port_by_af(options, AF_INET);
+}
+
+/** As router_get_advertised_or_port(), but allows an address family argument.
+ */
+uint16_t
+router_get_advertised_or_port_by_af(const or_options_t *options,
+ sa_family_t family)
+{
+ int port = get_first_advertised_port_by_type_af(CONN_TYPE_OR_LISTENER,
+ family);
+ (void)options;
+
+ /* If the port is in 'auto' mode, we have to use
+ router_get_listener_port_by_type(). */
+ if (port == CFG_AUTO_PORT)
+ return router_get_active_listener_port_by_type_af(CONN_TYPE_OR_LISTENER,
+ family);
+
+ return port;
+}
+
+/** Return the port that we should advertise as our DirPort;
+ * this is one of three possibilities:
+ * The one that is passed as <b>dirport</b> if the DirPort option is 0, or
+ * the one configured in the DirPort option,
+ * or the one we actually bound to if DirPort is "auto". */
+uint16_t
+router_get_advertised_dir_port(const or_options_t *options, uint16_t dirport)
+{
+ int dirport_configured = get_primary_dir_port();
+ (void)options;
+
+ if (!dirport_configured)
+ return dirport;
+
+ if (dirport_configured == CFG_AUTO_PORT)
+ return router_get_active_listener_port_by_type_af(CONN_TYPE_DIR_LISTENER,
+ AF_INET);
+
+ return dirport_configured;
+}
+
+/*
+ * OR descriptor generation.
+ */
+
+/** My routerinfo. */
+static routerinfo_t *desc_routerinfo = NULL;
+/** My extrainfo */
+static extrainfo_t *desc_extrainfo = NULL;
+/** Why did we most recently decide to regenerate our descriptor? Used to
+ * tell the authorities why we're sending it to them. */
+static const char *desc_gen_reason = "uninitialized reason";
+/** Since when has our descriptor been "clean"? 0 if we need to regenerate it
+ * now. */
+static time_t desc_clean_since = 0;
+/** Why did we mark the descriptor dirty? */
+static const char *desc_dirty_reason = "Tor just started";
+/** Boolean: do we need to regenerate the above? */
+static int desc_needs_upload = 0;
+
+/** OR only: If <b>force</b> is true, or we haven't uploaded this
+ * descriptor successfully yet, try to upload our signed descriptor to
+ * all the directory servers we know about.
+ */
+void
+router_upload_dir_desc_to_dirservers(int force)
+{
+ const routerinfo_t *ri;
+ extrainfo_t *ei;
+ char *msg;
+ size_t desc_len, extra_len = 0, total_len;
+ dirinfo_type_t auth = get_options()->PublishServerDescriptor_;
+
+ ri = router_get_my_routerinfo();
+ if (!ri) {
+ log_info(LD_GENERAL, "No descriptor; skipping upload");
+ return;
+ }
+ ei = router_get_my_extrainfo();
+ if (auth == NO_DIRINFO)
+ return;
+ if (!force && !desc_needs_upload)
+ return;
+
+ log_info(LD_OR, "Uploading relay descriptor to directory authorities%s",
+ force ? " (forced)" : "");
+
+ desc_needs_upload = 0;
+
+ desc_len = ri->cache_info.signed_descriptor_len;
+ extra_len = ei ? ei->cache_info.signed_descriptor_len : 0;
+ total_len = desc_len + extra_len + 1;
+ msg = tor_malloc(total_len);
+ memcpy(msg, ri->cache_info.signed_descriptor_body, desc_len);
+ if (ei) {
+ memcpy(msg+desc_len, ei->cache_info.signed_descriptor_body, extra_len);
+ }
+ msg[desc_len+extra_len] = 0;
+
+ directory_post_to_dirservers(DIR_PURPOSE_UPLOAD_DIR,
+ (auth & BRIDGE_DIRINFO) ?
+ ROUTER_PURPOSE_BRIDGE :
+ ROUTER_PURPOSE_GENERAL,
+ auth, msg, desc_len, extra_len);
+ tor_free(msg);
+}
+
+/** OR only: Check whether my exit policy says to allow connection to
+ * conn. Return 0 if we accept; non-0 if we reject.
+ */
+int
+router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port)
+{
+ const routerinfo_t *me = router_get_my_routerinfo();
+ if (!me) /* make sure routerinfo exists */
+ return -1;
+
+ /* make sure it's resolved to something. this way we can't get a
+ 'maybe' below. */
+ if (tor_addr_is_null(addr))
+ return -1;
+
+ /* look at router_get_my_routerinfo()->exit_policy for both the v4 and the
+ * v6 policies. The exit_policy field in router_get_my_routerinfo() is a
+ * bit unusual, in that it contains IPv6 and IPv6 entries. We don't want to
+ * look at router_get_my_routerinfo()->ipv6_exit_policy, since that's a port
+ * summary. */
+ if ((tor_addr_family(addr) == AF_INET ||
+ tor_addr_family(addr) == AF_INET6)) {
+ return compare_tor_addr_to_addr_policy(addr, port,
+ me->exit_policy) != ADDR_POLICY_ACCEPTED;
+#if 0
+ } else if (tor_addr_family(addr) == AF_INET6) {
+ return get_options()->IPv6Exit &&
+ desc_routerinfo->ipv6_exit_policy &&
+ compare_tor_addr_to_short_policy(addr, port,
+ me->ipv6_exit_policy) != ADDR_POLICY_ACCEPTED;
+#endif /* 0 */
+ } else {
+ return -1;
+ }
+}
+
+/** Return true iff my exit policy is reject *:*. Return -1 if we don't
+ * have a descriptor */
+MOCK_IMPL(int,
+router_my_exit_policy_is_reject_star,(void))
+{
+ const routerinfo_t *me = router_get_my_routerinfo();
+ if (!me) /* make sure routerinfo exists */
+ return -1;
+
+ return me->policy_is_reject_star;
+}
+
+/** Return true iff I'm a server and <b>digest</b> is equal to
+ * my server identity key digest. */
+int
+router_digest_is_me(const char *digest)
+{
+ return (server_identitykey &&
+ tor_memeq(server_identitykey_digest, digest, DIGEST_LEN));
+}
+
+/** Return my identity digest. */
+const uint8_t *
+router_get_my_id_digest(void)
+{
+ return (const uint8_t *)server_identitykey_digest;
+}
+
+/** Return true iff I'm a server and <b>digest</b> is equal to
+ * my identity digest. */
+int
+router_extrainfo_digest_is_me(const char *digest)
+{
+ extrainfo_t *ei = router_get_my_extrainfo();
+ if (!ei)
+ return 0;
+
+ return tor_memeq(digest,
+ ei->cache_info.signed_descriptor_digest,
+ DIGEST_LEN);
+}
+
+/** A wrapper around router_digest_is_me(). */
+int
+router_is_me(const routerinfo_t *router)
+{
+ return router_digest_is_me(router->cache_info.identity_digest);
+}
+
+/** Return a routerinfo for this OR, rebuilding a fresh one if
+ * necessary. Return NULL on error, or if called on an OP. */
+MOCK_IMPL(const routerinfo_t *,
+router_get_my_routerinfo,(void))
+{
+ return router_get_my_routerinfo_with_err(NULL);
+}
+
+/** Return routerinfo of this OR. Rebuild it from
+ * scratch if needed. Set <b>*err</b> to 0 on success or to
+ * appropriate TOR_ROUTERINFO_ERROR_* value on failure.
+ */
+MOCK_IMPL(const routerinfo_t *,
+router_get_my_routerinfo_with_err,(int *err))
+{
+ if (!server_mode(get_options())) {
+ if (err)
+ *err = TOR_ROUTERINFO_ERROR_NOT_A_SERVER;
+
+ return NULL;
+ }
+
+ if (!desc_clean_since) {
+ int rebuild_err = router_rebuild_descriptor(0);
+ if (rebuild_err < 0) {
+ if (err)
+ *err = rebuild_err;
+
+ return NULL;
+ }
+ }
+
+ if (!desc_routerinfo) {
+ if (err)
+ *err = TOR_ROUTERINFO_ERROR_DESC_REBUILDING;
+
+ return NULL;
+ }
+
+ if (err)
+ *err = 0;
+
+ return desc_routerinfo;
+}
+
+/** OR only: Return a signed server descriptor for this OR, rebuilding a fresh
+ * one if necessary. Return NULL on error.
+ */
+const char *
+router_get_my_descriptor(void)
+{
+ const char *body;
+ const routerinfo_t *me = router_get_my_routerinfo();
+ if (! me)
+ return NULL;
+ tor_assert(me->cache_info.saved_location == SAVED_NOWHERE);
+ body = signed_descriptor_get_body(&me->cache_info);
+ /* Make sure this is nul-terminated. */
+ tor_assert(!body[me->cache_info.signed_descriptor_len]);
+ log_debug(LD_GENERAL,"my desc is '%s'", body);
+ return body;
+}
+
+/** Return the extrainfo document for this OR, or NULL if we have none.
+ * Rebuilt it (and the server descriptor) if necessary. */
+extrainfo_t *
+router_get_my_extrainfo(void)
+{
+ if (!server_mode(get_options()))
+ return NULL;
+ if (router_rebuild_descriptor(0))
+ return NULL;
+ return desc_extrainfo;
+}
+
+/** Return a human-readable string describing what triggered us to generate
+ * our current descriptor, or NULL if we don't know. */
+const char *
+router_get_descriptor_gen_reason(void)
+{
+ return desc_gen_reason;
+}
+
+/** A list of nicknames that we've warned about including in our family
+ * declaration verbatim rather than as digests. */
+static smartlist_t *warned_nonexistent_family = NULL;
+
+static int router_guess_address_from_dir_headers(uint32_t *guess);
+
+/** Make a current best guess at our address, either because
+ * it's configured in torrc, or because we've learned it from
+ * dirserver headers. Place the answer in *<b>addr</b> and return
+ * 0 on success, else return -1 if we have no guess.
+ *
+ * If <b>cache_only</b> is true, just return any cached answers, and
+ * don't try to get any new answers.
+ */
+MOCK_IMPL(int,
+router_pick_published_address,(const or_options_t *options, uint32_t *addr,
+ int cache_only))
+{
+ /* First, check the cached output from resolve_my_address(). */
+ *addr = get_last_resolved_addr();
+ if (*addr)
+ return 0;
+
+ /* Second, consider doing a resolve attempt right here. */
+ if (!cache_only) {
+ if (resolve_my_address(LOG_INFO, options, addr, NULL, NULL) >= 0) {
+ log_info(LD_CONFIG,"Success: chose address '%s'.", fmt_addr32(*addr));
+ return 0;
+ }
+ }
+
+ /* Third, check the cached output from router_new_address_suggestion(). */
+ if (router_guess_address_from_dir_headers(addr) >= 0)
+ return 0;
+
+ /* We have no useful cached answers. Return failure. */
+ return -1;
+}
+
+/* Like router_check_descriptor_address_consistency, but specifically for the
+ * ORPort or DirPort.
+ * listener_type is either CONN_TYPE_OR_LISTENER or CONN_TYPE_DIR_LISTENER. */
+static void
+router_check_descriptor_address_port_consistency(uint32_t ipv4h_desc_addr,
+ int listener_type)
+{
+ tor_assert(listener_type == CONN_TYPE_OR_LISTENER ||
+ listener_type == CONN_TYPE_DIR_LISTENER);
+
+ /* The first advertised Port may be the magic constant CFG_AUTO_PORT.
+ */
+ int port_v4_cfg = get_first_advertised_port_by_type_af(listener_type,
+ AF_INET);
+ if (port_v4_cfg != 0 &&
+ !port_exists_by_type_addr32h_port(listener_type,
+ ipv4h_desc_addr, port_v4_cfg, 1)) {
+ const tor_addr_t *port_addr = get_first_advertised_addr_by_type_af(
+ listener_type,
+ AF_INET);
+ /* If we're building a descriptor with no advertised address,
+ * something is terribly wrong. */
+ tor_assert(port_addr);
+
+ tor_addr_t desc_addr;
+ char port_addr_str[TOR_ADDR_BUF_LEN];
+ char desc_addr_str[TOR_ADDR_BUF_LEN];
+
+ tor_addr_to_str(port_addr_str, port_addr, TOR_ADDR_BUF_LEN, 0);
+
+ tor_addr_from_ipv4h(&desc_addr, ipv4h_desc_addr);
+ tor_addr_to_str(desc_addr_str, &desc_addr, TOR_ADDR_BUF_LEN, 0);
+
+ const char *listener_str = (listener_type == CONN_TYPE_OR_LISTENER ?
+ "OR" : "Dir");
+ log_warn(LD_CONFIG, "The IPv4 %sPort address %s does not match the "
+ "descriptor address %s. If you have a static public IPv4 "
+ "address, use 'Address <IPv4>' and 'OutboundBindAddress "
+ "<IPv4>'. If you are behind a NAT, use two %sPort lines: "
+ "'%sPort <PublicPort> NoListen' and '%sPort <InternalPort> "
+ "NoAdvertise'.",
+ listener_str, port_addr_str, desc_addr_str, listener_str,
+ listener_str, listener_str);
+ }
+}
+
+/* Tor relays only have one IPv4 address in the descriptor, which is derived
+ * from the Address torrc option, or guessed using various methods in
+ * router_pick_published_address().
+ * Warn the operator if there is no ORPort on the descriptor address
+ * ipv4h_desc_addr.
+ * Warn the operator if there is no DirPort on the descriptor address.
+ * This catches a few common config errors:
+ * - operators who expect ORPorts and DirPorts to be advertised on the
+ * ports' listen addresses, rather than the torrc Address (or guessed
+ * addresses in the absence of an Address config). This includes
+ * operators who attempt to put their ORPort and DirPort on different
+ * addresses;
+ * - discrepancies between guessed addresses and configured listen
+ * addresses (when the Address option isn't set).
+ * If a listener is listening on all IPv4 addresses, it is assumed that it
+ * is listening on the configured Address, and no messages are logged.
+ * If an operators has specified NoAdvertise ORPorts in a NAT setting,
+ * no messages are logged, unless they have specified other advertised
+ * addresses.
+ * The message tells operators to configure an ORPort and DirPort that match
+ * the Address (using NoListen if needed).
+ */
+static void
+router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr)
+{
+ router_check_descriptor_address_port_consistency(ipv4h_desc_addr,
+ CONN_TYPE_OR_LISTENER);
+ router_check_descriptor_address_port_consistency(ipv4h_desc_addr,
+ CONN_TYPE_DIR_LISTENER);
+}
+
+/** Build a fresh routerinfo, signed server descriptor, and extra-info document
+ * for this OR. Set r to the generated routerinfo, e to the generated
+ * extra-info document. Return 0 on success, -1 on temporary error. Failure to
+ * generate an extra-info document is not an error and is indicated by setting
+ * e to NULL. Caller is responsible for freeing generated documents if 0 is
+ * returned.
+ */
+int
+router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
+{
+ routerinfo_t *ri;
+ extrainfo_t *ei;
+ uint32_t addr;
+ char platform[256];
+ int hibernating = we_are_hibernating();
+ const or_options_t *options = get_options();
+
+ if (router_pick_published_address(options, &addr, 0) < 0) {
+ log_warn(LD_CONFIG, "Don't know my address while generating descriptor");
+ return TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
+ }
+
+ /* Log a message if the address in the descriptor doesn't match the ORPort
+ * and DirPort addresses configured by the operator. */
+ router_check_descriptor_address_consistency(addr);
+
+ ri = tor_malloc_zero(sizeof(routerinfo_t));
+ ri->cache_info.routerlist_index = -1;
+ ri->nickname = tor_strdup(options->Nickname);
+ ri->addr = addr;
+ ri->or_port = router_get_advertised_or_port(options);
+ ri->dir_port = router_get_advertised_dir_port(options, 0);
+ ri->supports_tunnelled_dir_requests =
+ directory_permits_begindir_requests(options);
+ ri->cache_info.published_on = time(NULL);
+ /* get_onion_key() must invoke from main thread */
+ router_set_rsa_onion_pkey(get_onion_key(), &ri->onion_pkey,
+ &ri->onion_pkey_len);
+
+ ri->onion_curve25519_pkey =
+ tor_memdup(&get_current_curve25519_keypair()->pubkey,
+ sizeof(curve25519_public_key_t));
+
+ /* For now, at most one IPv6 or-address is being advertised. */
+ {
+ const port_cfg_t *ipv6_orport = NULL;
+ SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) {
+ if (p->type == CONN_TYPE_OR_LISTENER &&
+ ! p->server_cfg.no_advertise &&
+ ! p->server_cfg.bind_ipv4_only &&
+ tor_addr_family(&p->addr) == AF_INET6) {
+ /* Like IPv4, if the relay is configured using the default
+ * authorities, disallow internal IPs. Otherwise, allow them. */
+ const int default_auth = using_default_dir_authorities(options);
+ if (! tor_addr_is_internal(&p->addr, 0) || ! default_auth) {
+ ipv6_orport = p;
+ break;
+ } else {
+ char addrbuf[TOR_ADDR_BUF_LEN];
+ log_warn(LD_CONFIG,
+ "Unable to use configured IPv6 address \"%s\" in a "
+ "descriptor. Skipping it. "
+ "Try specifying a globally reachable address explicitly.",
+ tor_addr_to_str(addrbuf, &p->addr, sizeof(addrbuf), 1));
+ }
+ }
+ } SMARTLIST_FOREACH_END(p);
+ if (ipv6_orport) {
+ tor_addr_copy(&ri->ipv6_addr, &ipv6_orport->addr);
+ ri->ipv6_orport = ipv6_orport->port;
+ }
+ }
+
+ ri->identity_pkey = crypto_pk_dup_key(get_server_identity_key());
+ if (BUG(crypto_pk_get_digest(ri->identity_pkey,
+ ri->cache_info.identity_digest) < 0)) {
+ routerinfo_free(ri);
+ return TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
+ }
+ ri->cache_info.signing_key_cert =
+ tor_cert_dup(get_master_signing_key_cert());
+
+ get_platform_str(platform, sizeof(platform));
+ ri->platform = tor_strdup(platform);
+
+ ri->protocol_list = tor_strdup(protover_get_supported_protocols());
+
+ /* compute ri->bandwidthrate as the min of various options */
+ ri->bandwidthrate = get_effective_bwrate(options);
+
+ /* and compute ri->bandwidthburst similarly */
+ ri->bandwidthburst = get_effective_bwburst(options);
+
+ /* Report bandwidth, unless we're hibernating or shutting down */
+ ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess();
+
+ if (dns_seems_to_be_broken() || has_dns_init_failed()) {
+ /* DNS is screwed up; don't claim to be an exit. */
+ policies_exit_policy_append_reject_star(&ri->exit_policy);
+ } else {
+ policies_parse_exit_policy_from_options(options,ri->addr,&ri->ipv6_addr,
+ &ri->exit_policy);
+ }
+ ri->policy_is_reject_star =
+ policy_is_reject_star(ri->exit_policy, AF_INET, 1) &&
+ policy_is_reject_star(ri->exit_policy, AF_INET6, 1);
+
+ if (options->IPv6Exit) {
+ char *p_tmp = policy_summarize(ri->exit_policy, AF_INET6);
+ if (p_tmp)
+ ri->ipv6_exit_policy = parse_short_policy(p_tmp);
+ tor_free(p_tmp);
+ }
+
+ if (options->MyFamily && ! options->BridgeRelay) {
+ if (!warned_nonexistent_family)
+ warned_nonexistent_family = smartlist_new();
+ ri->declared_family = smartlist_new();
+ config_line_t *family;
+ for (family = options->MyFamily; family; family = family->next) {
+ char *name = family->value;
+ const node_t *member;
+ if (!strcasecmp(name, options->Nickname))
+ continue; /* Don't list ourself, that's redundant */
+ else
+ member = node_get_by_nickname(name, 0);
+ if (!member) {
+ int is_legal = is_legal_nickname_or_hexdigest(name);
+ if (!smartlist_contains_string(warned_nonexistent_family, name) &&
+ !is_legal_hexdigest(name)) {
+ if (is_legal)
+ log_warn(LD_CONFIG,
+ "I have no descriptor for the router named \"%s\" in my "
+ "declared family; I'll use the nickname as is, but "
+ "this may confuse clients.", name);
+ else
+ log_warn(LD_CONFIG, "There is a router named \"%s\" in my "
+ "declared family, but that isn't a legal nickname. "
+ "Skipping it.", escaped(name));
+ smartlist_add_strdup(warned_nonexistent_family, name);
+ }
+ if (is_legal) {
+ smartlist_add_strdup(ri->declared_family, name);
+ }
+ } else if (router_digest_is_me(member->identity)) {
+ /* Don't list ourself in our own family; that's redundant */
+ /* XXX shouldn't be possible */
+ } else {
+ char *fp = tor_malloc(HEX_DIGEST_LEN+2);
+ fp[0] = '$';
+ base16_encode(fp+1,HEX_DIGEST_LEN+1,
+ member->identity, DIGEST_LEN);
+ smartlist_add(ri->declared_family, fp);
+ if (smartlist_contains_string(warned_nonexistent_family, name))
+ smartlist_string_remove(warned_nonexistent_family, name);
+ }
+ }
+
+ /* remove duplicates from the list */
+ smartlist_sort_strings(ri->declared_family);
+ smartlist_uniq_strings(ri->declared_family);
+ }
+
+ /* Now generate the extrainfo. */
+ ei = tor_malloc_zero(sizeof(extrainfo_t));
+ ei->cache_info.is_extrainfo = 1;
+ strlcpy(ei->nickname, get_options()->Nickname, sizeof(ei->nickname));
+ ei->cache_info.published_on = ri->cache_info.published_on;
+ ei->cache_info.signing_key_cert =
+ tor_cert_dup(get_master_signing_key_cert());
+
+ memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest,
+ DIGEST_LEN);
+ if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body,
+ ei, get_server_identity_key(),
+ get_master_signing_keypair()) < 0) {
+ log_warn(LD_BUG, "Couldn't generate extra-info descriptor.");
+ extrainfo_free(ei);
+ ei = NULL;
+ } else {
+ ei->cache_info.signed_descriptor_len =
+ strlen(ei->cache_info.signed_descriptor_body);
+ router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body,
+ ei->cache_info.signed_descriptor_len,
+ ei->cache_info.signed_descriptor_digest);
+ crypto_digest256((char*) ei->digest256,
+ ei->cache_info.signed_descriptor_body,
+ ei->cache_info.signed_descriptor_len,
+ DIGEST_SHA256);
+ }
+
+ /* Now finish the router descriptor. */
+ if (ei) {
+ memcpy(ri->cache_info.extra_info_digest,
+ ei->cache_info.signed_descriptor_digest,
+ DIGEST_LEN);
+ memcpy(ri->cache_info.extra_info_digest256,
+ ei->digest256,
+ DIGEST256_LEN);
+ } else {
+ /* ri was allocated with tor_malloc_zero, so there is no need to
+ * zero ri->cache_info.extra_info_digest here. */
+ }
+ if (! (ri->cache_info.signed_descriptor_body =
+ router_dump_router_to_string(ri, get_server_identity_key(),
+ get_onion_key(),
+ get_current_curve25519_keypair(),
+ get_master_signing_keypair())) ) {
+ log_warn(LD_BUG, "Couldn't generate router descriptor.");
+ routerinfo_free(ri);
+ extrainfo_free(ei);
+ return TOR_ROUTERINFO_ERROR_CANNOT_GENERATE;
+ }
+ ri->cache_info.signed_descriptor_len =
+ strlen(ri->cache_info.signed_descriptor_body);
+
+ ri->purpose =
+ options->BridgeRelay ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
+ if (options->BridgeRelay) {
+ /* Bridges shouldn't be able to send their descriptors unencrypted,
+ anyway, since they don't have a DirPort, and always connect to the
+ bridge authority anonymously. But just in case they somehow think of
+ sending them on an unencrypted connection, don't allow them to try. */
+ ri->cache_info.send_unencrypted = 0;
+ if (ei)
+ ei->cache_info.send_unencrypted = 0;
+ } else {
+ ri->cache_info.send_unencrypted = 1;
+ if (ei)
+ ei->cache_info.send_unencrypted = 1;
+ }
+
+ router_get_router_hash(ri->cache_info.signed_descriptor_body,
+ strlen(ri->cache_info.signed_descriptor_body),
+ ri->cache_info.signed_descriptor_digest);
+
+ if (ei) {
+ tor_assert(!
+ routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
+ &ri->cache_info, NULL));
+ }
+
+ *r = ri;
+ *e = ei;
+ return 0;
+}
+
+/** If <b>force</b> is true, or our descriptor is out-of-date, rebuild a fresh
+ * routerinfo, signed server descriptor, and extra-info document for this OR.
+ * Return 0 on success, -1 on temporary error.
+ */
+int
+router_rebuild_descriptor(int force)
+{
+ int err = 0;
+ routerinfo_t *ri;
+ extrainfo_t *ei;
+ uint32_t addr;
+ const or_options_t *options = get_options();
+
+ if (desc_clean_since && !force)
+ return 0;
+
+ if (router_pick_published_address(options, &addr, 0) < 0 ||
+ router_get_advertised_or_port(options) == 0) {
+ /* Stop trying to rebuild our descriptor every second. We'll
+ * learn that it's time to try again when ip_address_changed()
+ * marks it dirty. */
+ desc_clean_since = time(NULL);
+ return TOR_ROUTERINFO_ERROR_DESC_REBUILDING;
+ }
+
+ log_info(LD_OR, "Rebuilding relay descriptor%s", force ? " (forced)" : "");
+
+ err = router_build_fresh_descriptor(&ri, &ei);
+ if (err < 0) {
+ return err;
+ }
+
+ routerinfo_free(desc_routerinfo);
+ desc_routerinfo = ri;
+ extrainfo_free(desc_extrainfo);
+ desc_extrainfo = ei;
+
+ desc_clean_since = time(NULL);
+ desc_needs_upload = 1;
+ desc_gen_reason = desc_dirty_reason;
+ if (BUG(desc_gen_reason == NULL)) {
+ desc_gen_reason = "descriptor was marked dirty earlier, for no reason.";
+ }
+ desc_dirty_reason = NULL;
+ control_event_my_descriptor_changed();
+ return 0;
+}
+
+/** If our router descriptor ever goes this long without being regenerated
+ * because something changed, we force an immediate regenerate-and-upload. */
+#define FORCE_REGENERATE_DESCRIPTOR_INTERVAL (18*60*60)
+
+/** If our router descriptor seems to be missing or unacceptable according
+ * to the authorities, regenerate and reupload it _this_ often. */
+#define FAST_RETRY_DESCRIPTOR_INTERVAL (90*60)
+
+/** Mark descriptor out of date if it's been "too long" since we last tried
+ * to upload one. */
+void
+mark_my_descriptor_dirty_if_too_old(time_t now)
+{
+ networkstatus_t *ns;
+ const routerstatus_t *rs;
+ const char *retry_fast_reason = NULL; /* Set if we should retry frequently */
+ const time_t slow_cutoff = now - FORCE_REGENERATE_DESCRIPTOR_INTERVAL;
+ const time_t fast_cutoff = now - FAST_RETRY_DESCRIPTOR_INTERVAL;
+
+ /* If it's already dirty, don't mark it. */
+ if (! desc_clean_since)
+ return;
+
+ /* If it's older than FORCE_REGENERATE_DESCRIPTOR_INTERVAL, it's always
+ * time to rebuild it. */
+ if (desc_clean_since < slow_cutoff) {
+ mark_my_descriptor_dirty("time for new descriptor");
+ return;
+ }
+ /* Now we see whether we want to be retrying frequently or no. The
+ * rule here is that we'll retry frequently if we aren't listed in the
+ * live consensus we have, or if the publication time of the
+ * descriptor listed for us in the consensus is very old. */
+ ns = networkstatus_get_live_consensus(now);
+ if (ns) {
+ rs = networkstatus_vote_find_entry(ns, server_identitykey_digest);
+ if (rs == NULL)
+ retry_fast_reason = "not listed in consensus";
+ else if (rs->published_on < slow_cutoff)
+ retry_fast_reason = "version listed in consensus is quite old";
+ }
+
+ if (retry_fast_reason && desc_clean_since < fast_cutoff)
+ mark_my_descriptor_dirty(retry_fast_reason);
+}
+
+/** Call when the current descriptor is out of date. */
+void
+mark_my_descriptor_dirty(const char *reason)
+{
+ const or_options_t *options = get_options();
+ if (BUG(reason == NULL)) {
+ reason = "marked descriptor dirty for unspecified reason";
+ }
+ if (server_mode(options) && options->PublishServerDescriptor_)
+ log_info(LD_OR, "Decided to publish new relay descriptor: %s", reason);
+ desc_clean_since = 0;
+ if (!desc_dirty_reason)
+ desc_dirty_reason = reason;
+}
+
+/** How frequently will we republish our descriptor because of large (factor
+ * of 2) shifts in estimated bandwidth? Note: We don't use this constant
+ * if our previous bandwidth estimate was exactly 0. */
+#define MAX_BANDWIDTH_CHANGE_FREQ (3*60*60)
+
++/** Maximum uptime to republish our descriptor because of large shifts in
++ * estimated bandwidth. */
++#define MAX_UPTIME_BANDWIDTH_CHANGE (24*60*60)
++
++/** By which factor bandwidth shifts have to change to be considered large. */
++#define BANDWIDTH_CHANGE_FACTOR 2
++
+/** Check whether bandwidth has changed a lot since the last time we announced
- * bandwidth. If so, mark our descriptor dirty. */
++ * bandwidth while the uptime is smaller than MAX_UPTIME_BANDWIDTH_CHANGE.
++ * If so, mark our descriptor dirty. */
+void
+check_descriptor_bandwidth_changed(time_t now)
+{
+ static time_t last_changed = 0;
+ uint64_t prev, cur;
++ const int hibernating = we_are_hibernating();
++
++ /* If the relay uptime is bigger than MAX_UPTIME_BANDWIDTH_CHANGE,
++ * the next regularly scheduled descriptor update (18h) will be enough */
++ if (get_uptime() > MAX_UPTIME_BANDWIDTH_CHANGE && !hibernating)
++ return;
++
+ const routerinfo_t *my_ri = router_get_my_routerinfo();
- if (!my_ri) /* make sure routerinfo exists */
++
++ if (!my_ri)
+ return;
+
+ prev = my_ri->bandwidthcapacity;
+
+ /* Consider ourselves to have zero bandwidth if we're hibernating or
+ * shutting down. */
- cur = we_are_hibernating() ? 0 : rep_hist_bandwidth_assess();
++ cur = hibernating ? 0 : rep_hist_bandwidth_assess();
++
+ if ((prev != cur && (!prev || !cur)) ||
- cur > prev*2 ||
- cur < prev/2) {
++ cur > (prev * BANDWIDTH_CHANGE_FACTOR) ||
++ cur < (prev / BANDWIDTH_CHANGE_FACTOR) ) {
+ if (last_changed+MAX_BANDWIDTH_CHANGE_FREQ < now || !prev) {
+ log_info(LD_GENERAL,
+ "Measured bandwidth has changed; rebuilding descriptor.");
+ mark_my_descriptor_dirty("bandwidth has changed");
+ last_changed = now;
+ }
+ }
+}
+
+/** Note at log level severity that our best guess of address has changed from
+ * <b>prev</b> to <b>cur</b>. */
+static void
+log_addr_has_changed(int severity,
+ const tor_addr_t *prev,
+ const tor_addr_t *cur,
+ const char *source)
+{
+ char addrbuf_prev[TOR_ADDR_BUF_LEN];
+ char addrbuf_cur[TOR_ADDR_BUF_LEN];
+
+ if (BUG(!server_mode(get_options())))
+ return;
+
+ if (tor_addr_to_str(addrbuf_prev, prev, sizeof(addrbuf_prev), 1) == NULL)
+ strlcpy(addrbuf_prev, "???", TOR_ADDR_BUF_LEN);
+ if (tor_addr_to_str(addrbuf_cur, cur, sizeof(addrbuf_cur), 1) == NULL)
+ strlcpy(addrbuf_cur, "???", TOR_ADDR_BUF_LEN);
+
+ if (!tor_addr_is_null(prev))
+ log_fn(severity, LD_GENERAL,
+ "Our IP Address has changed from %s to %s; "
+ "rebuilding descriptor (source: %s).",
+ addrbuf_prev, addrbuf_cur, source);
+ else
+ log_notice(LD_GENERAL,
+ "Guessed our IP address as %s (source: %s).",
+ addrbuf_cur, source);
+}
+
+/** Check whether our own address as defined by the Address configuration
+ * has changed. This is for routers that get their address from a service
+ * like dyndns. If our address has changed, mark our descriptor dirty. */
+void
+check_descriptor_ipaddress_changed(time_t now)
+{
+ uint32_t prev, cur;
+ const or_options_t *options = get_options();
+ const char *method = NULL;
+ char *hostname = NULL;
+ const routerinfo_t *my_ri = router_get_my_routerinfo();
+
+ (void) now;
+
+ if (my_ri == NULL) /* make sure routerinfo exists */
+ return;
+
+ /* XXXX ipv6 */
+ prev = my_ri->addr;
+ if (resolve_my_address(LOG_INFO, options, &cur, &method, &hostname) < 0) {
+ log_info(LD_CONFIG,"options->Address didn't resolve into an IP.");
+ return;
+ }
+
+ if (prev != cur) {
+ char *source;
+ tor_addr_t tmp_prev, tmp_cur;
+
+ tor_addr_from_ipv4h(&tmp_prev, prev);
+ tor_addr_from_ipv4h(&tmp_cur, cur);
+
+ tor_asprintf(&source, "METHOD=%s%s%s", method,
+ hostname ? " HOSTNAME=" : "",
+ hostname ? hostname : "");
+
+ log_addr_has_changed(LOG_NOTICE, &tmp_prev, &tmp_cur, source);
+ tor_free(source);
+
+ ip_address_changed(0);
+ }
+
+ tor_free(hostname);
+}
+
+/** The most recently guessed value of our IP address, based on directory
+ * headers. */
+static tor_addr_t last_guessed_ip = TOR_ADDR_NULL;
+
+/** A directory server <b>d_conn</b> told us our IP address is
+ * <b>suggestion</b>.
+ * If this address is different from the one we think we are now, and
+ * if our computer doesn't actually know its IP address, then switch. */
+void
+router_new_address_suggestion(const char *suggestion,
+ const dir_connection_t *d_conn)
+{
+ tor_addr_t addr;
+ uint32_t cur = 0; /* Current IPv4 address. */
+ const or_options_t *options = get_options();
+
+ /* first, learn what the IP address actually is */
+ if (tor_addr_parse(&addr, suggestion) == -1) {
+ log_debug(LD_DIR, "Malformed X-Your-Address-Is header %s. Ignoring.",
+ escaped(suggestion));
+ return;
+ }
+
+ log_debug(LD_DIR, "Got X-Your-Address-Is: %s.", suggestion);
+
+ if (!server_mode(options)) {
+ tor_addr_copy(&last_guessed_ip, &addr);
+ return;
+ }
+
+ /* XXXX ipv6 */
+ cur = get_last_resolved_addr();
+ if (cur ||
+ resolve_my_address(LOG_INFO, options, &cur, NULL, NULL) >= 0) {
+ /* We're all set -- we already know our address. Great. */
+ tor_addr_from_ipv4h(&last_guessed_ip, cur); /* store it in case we
+ need it later */
+ return;
+ }
+ if (tor_addr_is_internal(&addr, 0)) {
+ /* Don't believe anybody who says our IP is, say, 127.0.0.1. */
+ return;
+ }
+ if (tor_addr_eq(&d_conn->base_.addr, &addr)) {
+ /* Don't believe anybody who says our IP is their IP. */
+ log_debug(LD_DIR, "A directory server told us our IP address is %s, "
+ "but they are just reporting their own IP address. Ignoring.",
+ suggestion);
+ return;
+ }
+
+ /* Okay. We can't resolve our own address, and X-Your-Address-Is is giving
+ * us an answer different from what we had the last time we managed to
+ * resolve it. */
+ if (!tor_addr_eq(&last_guessed_ip, &addr)) {
+ control_event_server_status(LOG_NOTICE,
+ "EXTERNAL_ADDRESS ADDRESS=%s METHOD=DIRSERV",
+ suggestion);
+ log_addr_has_changed(LOG_NOTICE, &last_guessed_ip, &addr,
+ d_conn->base_.address);
+ ip_address_changed(0);
+ tor_addr_copy(&last_guessed_ip, &addr); /* router_rebuild_descriptor()
+ will fetch it */
+ }
+}
+
+/** We failed to resolve our address locally, but we'd like to build
+ * a descriptor and publish / test reachability. If we have a guess
+ * about our address based on directory headers, answer it and return
+ * 0; else return -1. */
+static int
+router_guess_address_from_dir_headers(uint32_t *guess)
+{
+ if (!tor_addr_is_null(&last_guessed_ip)) {
+ *guess = tor_addr_to_ipv4h(&last_guessed_ip);
+ return 0;
+ }
+ return -1;
+}
+
+/** Set <b>platform</b> (max length <b>len</b>) to a NUL-terminated short
+ * string describing the version of Tor and the operating system we're
+ * currently running on.
+ */
+STATIC void
+get_platform_str(char *platform, size_t len)
+{
+ tor_snprintf(platform, len, "Tor %s on %s",
+ get_short_version(), get_uname());
+}
+
+/* XXX need to audit this thing and count fenceposts. maybe
+ * refactor so we don't have to keep asking if we're
+ * near the end of maxlen?
+ */
+#define DEBUG_ROUTER_DUMP_ROUTER_TO_STRING
+
+/** OR only: Given a routerinfo for this router, and an identity key to sign
+ * with, encode the routerinfo as a signed server descriptor and return a new
+ * string encoding the result, or NULL on failure.
+ */
+char *
+router_dump_router_to_string(routerinfo_t *router,
+ const crypto_pk_t *ident_key,
+ const crypto_pk_t *tap_key,
+ const curve25519_keypair_t *ntor_keypair,
+ const ed25519_keypair_t *signing_keypair)
+{
+ char *address = NULL;
+ char *onion_pkey = NULL; /* Onion key, PEM-encoded. */
+ crypto_pk_t *rsa_pubkey = NULL;
+ char *identity_pkey = NULL; /* Identity key, PEM-encoded. */
+ char digest[DIGEST256_LEN];
+ char published[ISO_TIME_LEN+1];
+ char fingerprint[FINGERPRINT_LEN+1];
+ char *extra_info_line = NULL;
+ size_t onion_pkeylen, identity_pkeylen;
+ char *family_line = NULL;
+ char *extra_or_address = NULL;
+ const or_options_t *options = get_options();
+ smartlist_t *chunks = NULL;
+ char *output = NULL;
+ const int emit_ed_sigs = signing_keypair &&
+ router->cache_info.signing_key_cert;
+ char *ed_cert_line = NULL;
+ char *rsa_tap_cc_line = NULL;
+ char *ntor_cc_line = NULL;
+ char *proto_line = NULL;
+
+ /* Make sure the identity key matches the one in the routerinfo. */
+ if (!crypto_pk_eq_keys(ident_key, router->identity_pkey)) {
+ log_warn(LD_BUG,"Tried to sign a router with a private key that didn't "
+ "match router's public key!");
+ goto err;
+ }
+ if (emit_ed_sigs) {
+ if (!router->cache_info.signing_key_cert->signing_key_included ||
+ !ed25519_pubkey_eq(&router->cache_info.signing_key_cert->signed_key,
+ &signing_keypair->pubkey)) {
+ log_warn(LD_BUG, "Tried to sign a router descriptor with a mismatched "
+ "ed25519 key chain %d",
+ router->cache_info.signing_key_cert->signing_key_included);
+ goto err;
+ }
+ }
+
+ /* record our fingerprint, so we can include it in the descriptor */
+ if (crypto_pk_get_fingerprint(router->identity_pkey, fingerprint, 1)<0) {
+ log_err(LD_BUG,"Error computing fingerprint");
+ goto err;
+ }
+
+ if (emit_ed_sigs) {
+ /* Encode ed25519 signing cert */
+ char ed_cert_base64[256];
+ char ed_fp_base64[ED25519_BASE64_LEN+1];
+ if (base64_encode(ed_cert_base64, sizeof(ed_cert_base64),
+ (const char*)router->cache_info.signing_key_cert->encoded,
+ router->cache_info.signing_key_cert->encoded_len,
+ BASE64_ENCODE_MULTILINE) < 0) {
+ log_err(LD_BUG,"Couldn't base64-encode signing key certificate!");
+ goto err;
+ }
+ if (ed25519_public_to_base64(ed_fp_base64,
+ &router->cache_info.signing_key_cert->signing_key)<0) {
+ log_err(LD_BUG,"Couldn't base64-encode identity key\n");
+ goto err;
+ }
+ tor_asprintf(&ed_cert_line, "identity-ed25519\n"
+ "-----BEGIN ED25519 CERT-----\n"
+ "%s"
+ "-----END ED25519 CERT-----\n"
+ "master-key-ed25519 %s\n",
+ ed_cert_base64, ed_fp_base64);
+ }
+
+ /* PEM-encode the onion key */
+ rsa_pubkey = router_get_rsa_onion_pkey(router->onion_pkey,
+ router->onion_pkey_len);
+ if (crypto_pk_write_public_key_to_string(rsa_pubkey,
+ &onion_pkey,&onion_pkeylen)<0) {
+ log_warn(LD_BUG,"write onion_pkey to string failed!");
+ goto err;
+ }
+
+ /* PEM-encode the identity key */
+ if (crypto_pk_write_public_key_to_string(router->identity_pkey,
+ &identity_pkey,&identity_pkeylen)<0) {
+ log_warn(LD_BUG,"write identity_pkey to string failed!");
+ goto err;
+ }
+
+ /* Cross-certify with RSA key */
+ if (tap_key && router->cache_info.signing_key_cert &&
+ router->cache_info.signing_key_cert->signing_key_included) {
+ char buf[256];
+ int tap_cc_len = 0;
+ uint8_t *tap_cc =
+ make_tap_onion_key_crosscert(tap_key,
+ &router->cache_info.signing_key_cert->signing_key,
+ router->identity_pkey,
+ &tap_cc_len);
+ if (!tap_cc) {
+ log_warn(LD_BUG,"make_tap_onion_key_crosscert failed!");
+ goto err;
+ }
+
+ if (base64_encode(buf, sizeof(buf), (const char*)tap_cc, tap_cc_len,
+ BASE64_ENCODE_MULTILINE) < 0) {
+ log_warn(LD_BUG,"base64_encode(rsa_crosscert) failed!");
+ tor_free(tap_cc);
+ goto err;
+ }
+ tor_free(tap_cc);
+
+ tor_asprintf(&rsa_tap_cc_line,
+ "onion-key-crosscert\n"
+ "-----BEGIN CROSSCERT-----\n"
+ "%s"
+ "-----END CROSSCERT-----\n", buf);
+ }
+
+ /* Cross-certify with onion keys */
+ if (ntor_keypair && router->cache_info.signing_key_cert &&
+ router->cache_info.signing_key_cert->signing_key_included) {
+ int sign = 0;
+ char buf[256];
+ /* XXXX Base the expiration date on the actual onion key expiration time?*/
+ tor_cert_t *cert =
+ make_ntor_onion_key_crosscert(ntor_keypair,
+ &router->cache_info.signing_key_cert->signing_key,
+ router->cache_info.published_on,
+ get_onion_key_lifetime(), &sign);
+ if (!cert) {
+ log_warn(LD_BUG,"make_ntor_onion_key_crosscert failed!");
+ goto err;
+ }
+ tor_assert(sign == 0 || sign == 1);
+
+ if (base64_encode(buf, sizeof(buf),
+ (const char*)cert->encoded, cert->encoded_len,
+ BASE64_ENCODE_MULTILINE)<0) {
+ log_warn(LD_BUG,"base64_encode(ntor_crosscert) failed!");
+ tor_cert_free(cert);
+ goto err;
+ }
+ tor_cert_free(cert);
+
+ tor_asprintf(&ntor_cc_line,
+ "ntor-onion-key-crosscert %d\n"
+ "-----BEGIN ED25519 CERT-----\n"
+ "%s"
+ "-----END ED25519 CERT-----\n", sign, buf);
+ }
+
+ /* Encode the publication time. */
+ format_iso_time(published, router->cache_info.published_on);
+
+ if (router->declared_family && smartlist_len(router->declared_family)) {
+ char *family = smartlist_join_strings(router->declared_family,
+ " ", 0, NULL);
+ tor_asprintf(&family_line, "family %s\n", family);
+ tor_free(family);
+ } else {
+ family_line = tor_strdup("");
+ }
+
+ if (!tor_digest_is_zero(router->cache_info.extra_info_digest)) {
+ char extra_info_digest[HEX_DIGEST_LEN+1];
+ base16_encode(extra_info_digest, sizeof(extra_info_digest),
+ router->cache_info.extra_info_digest, DIGEST_LEN);
+ if (!tor_digest256_is_zero(router->cache_info.extra_info_digest256)) {
+ char d256_64[BASE64_DIGEST256_LEN+1];
+ digest256_to_base64(d256_64, router->cache_info.extra_info_digest256);
+ tor_asprintf(&extra_info_line, "extra-info-digest %s %s\n",
+ extra_info_digest, d256_64);
+ } else {
+ tor_asprintf(&extra_info_line, "extra-info-digest %s\n",
+ extra_info_digest);
+ }
+ }
+
+ if (router->ipv6_orport &&
+ tor_addr_family(&router->ipv6_addr) == AF_INET6) {
+ char addr[TOR_ADDR_BUF_LEN];
+ const char *a;
+ a = tor_addr_to_str(addr, &router->ipv6_addr, sizeof(addr), 1);
+ if (a) {
+ tor_asprintf(&extra_or_address,
+ "or-address %s:%d\n", a, router->ipv6_orport);
+ log_debug(LD_OR, "My or-address line is <%s>", extra_or_address);
+ }
+ }
+
+ if (router->protocol_list) {
+ tor_asprintf(&proto_line, "proto %s\n", router->protocol_list);
+ } else {
+ proto_line = tor_strdup("");
+ }
+
+ address = tor_dup_ip(router->addr);
+ chunks = smartlist_new();
+
+ /* Generate the easy portion of the router descriptor. */
+ smartlist_add_asprintf(chunks,
+ "router %s %s %d 0 %d\n"
+ "%s"
+ "%s"
+ "platform %s\n"
+ "%s"
+ "published %s\n"
+ "fingerprint %s\n"
+ "uptime %ld\n"
+ "bandwidth %d %d %d\n"
+ "%s%s"
+ "onion-key\n%s"
+ "signing-key\n%s"
+ "%s%s"
+ "%s%s%s",
+ router->nickname,
+ address,
+ router->or_port,
+ router_should_advertise_dirport(options, router->dir_port),
+ ed_cert_line ? ed_cert_line : "",
+ extra_or_address ? extra_or_address : "",
+ router->platform,
+ proto_line,
+ published,
+ fingerprint,
+ get_uptime(),
+ (int) router->bandwidthrate,
+ (int) router->bandwidthburst,
+ (int) router->bandwidthcapacity,
+ extra_info_line ? extra_info_line : "",
+ (options->DownloadExtraInfo || options->V3AuthoritativeDir) ?
+ "caches-extra-info\n" : "",
+ onion_pkey, identity_pkey,
+ rsa_tap_cc_line ? rsa_tap_cc_line : "",
+ ntor_cc_line ? ntor_cc_line : "",
+ family_line,
+ we_are_hibernating() ? "hibernating 1\n" : "",
+ "hidden-service-dir\n");
+
+ if (options->ContactInfo && strlen(options->ContactInfo)) {
+ const char *ci = options->ContactInfo;
+ if (strchr(ci, '\n') || strchr(ci, '\r'))
+ ci = escaped(ci);
+ smartlist_add_asprintf(chunks, "contact %s\n", ci);
+ }
+
+ if (options->BridgeRelay) {
+ const char *bd;
+ if (options->BridgeDistribution && strlen(options->BridgeDistribution)) {
+ bd = options->BridgeDistribution;
+ } else {
+ bd = "any";
+ }
+ if (strchr(bd, '\n') || strchr(bd, '\r'))
+ bd = escaped(bd);
+ smartlist_add_asprintf(chunks, "bridge-distribution-request %s\n", bd);
+ }
+
+ if (router->onion_curve25519_pkey) {
+ char kbuf[128];
+ base64_encode(kbuf, sizeof(kbuf),
+ (const char *)router->onion_curve25519_pkey->public_key,
+ CURVE25519_PUBKEY_LEN, BASE64_ENCODE_MULTILINE);
+ smartlist_add_asprintf(chunks, "ntor-onion-key %s", kbuf);
+ } else {
+ /* Authorities will start rejecting relays without ntor keys in 0.2.9 */
+ log_err(LD_BUG, "A relay must have an ntor onion key");
+ goto err;
+ }
+
+ /* Write the exit policy to the end of 's'. */
+ if (!router->exit_policy || !smartlist_len(router->exit_policy)) {
+ smartlist_add_strdup(chunks, "reject *:*\n");
+ } else if (router->exit_policy) {
+ char *exit_policy = router_dump_exit_policy_to_string(router,1,0);
+
+ if (!exit_policy)
+ goto err;
+
+ smartlist_add_asprintf(chunks, "%s\n", exit_policy);
+ tor_free(exit_policy);
+ }
+
+ if (router->ipv6_exit_policy) {
+ char *p6 = write_short_policy(router->ipv6_exit_policy);
+ if (p6 && strcmp(p6, "reject 1-65535")) {
+ smartlist_add_asprintf(chunks,
+ "ipv6-policy %s\n", p6);
+ }
+ tor_free(p6);
+ }
+
+ if (router_should_advertise_begindir(options,
+ router->supports_tunnelled_dir_requests)) {
+ smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
+ }
+
+ /* Sign the descriptor with Ed25519 */
+ if (emit_ed_sigs) {
+ smartlist_add_strdup(chunks, "router-sig-ed25519 ");
+ crypto_digest_smartlist_prefix(digest, DIGEST256_LEN,
+ ED_DESC_SIGNATURE_PREFIX,
+ chunks, "", DIGEST_SHA256);
+ ed25519_signature_t sig;
+ char buf[ED25519_SIG_BASE64_LEN+1];
+ if (ed25519_sign(&sig, (const uint8_t*)digest, DIGEST256_LEN,
+ signing_keypair) < 0)
+ goto err;
+ if (ed25519_signature_to_base64(buf, &sig) < 0)
+ goto err;
+
+ smartlist_add_asprintf(chunks, "%s\n", buf);
+ }
+
+ /* Sign the descriptor with RSA */
+ smartlist_add_strdup(chunks, "router-signature\n");
+
+ crypto_digest_smartlist(digest, DIGEST_LEN, chunks, "", DIGEST_SHA1);
+
+ {
+ char *sig;
+ if (!(sig = router_get_dirobj_signature(digest, DIGEST_LEN, ident_key))) {
+ log_warn(LD_BUG, "Couldn't sign router descriptor");
+ goto err;
+ }
+ smartlist_add(chunks, sig);
+ }
+
+ /* include a last '\n' */
+ smartlist_add_strdup(chunks, "\n");
+
+ output = smartlist_join_strings(chunks, "", 0, NULL);
+
+#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING
+ {
+ char *s_dup;
+ const char *cp;
+ routerinfo_t *ri_tmp;
+ cp = s_dup = tor_strdup(output);
+ ri_tmp = router_parse_entry_from_string(cp, NULL, 1, 0, NULL, NULL);
+ if (!ri_tmp) {
+ log_err(LD_BUG,
+ "We just generated a router descriptor we can't parse.");
+ log_err(LD_BUG, "Descriptor was: <<%s>>", output);
+ goto err;
+ }
+ tor_free(s_dup);
+ routerinfo_free(ri_tmp);
+ }
+#endif /* defined(DEBUG_ROUTER_DUMP_ROUTER_TO_STRING) */
+
+ goto done;
+
+ err:
+ tor_free(output); /* sets output to NULL */
+ done:
+ if (chunks) {
+ SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+ smartlist_free(chunks);
+ }
+ crypto_pk_free(rsa_pubkey);
+ tor_free(address);
+ tor_free(family_line);
+ tor_free(onion_pkey);
+ tor_free(identity_pkey);
+ tor_free(extra_or_address);
+ tor_free(ed_cert_line);
+ tor_free(rsa_tap_cc_line);
+ tor_free(ntor_cc_line);
+ tor_free(extra_info_line);
+ tor_free(proto_line);
+
+ return output;
+}
+
+/**
+ * OR only: Given <b>router</b>, produce a string with its exit policy.
+ * If <b>include_ipv4</b> is true, include IPv4 entries.
+ * If <b>include_ipv6</b> is true, include IPv6 entries.
+ */
+char *
+router_dump_exit_policy_to_string(const routerinfo_t *router,
+ int include_ipv4,
+ int include_ipv6)
+{
+ if ((!router->exit_policy) || (router->policy_is_reject_star)) {
+ return tor_strdup("reject *:*");
+ }
+
+ return policy_dump_to_string(router->exit_policy,
+ include_ipv4,
+ include_ipv6);
+}
+
+/** Copy the primary (IPv4) OR port (IP address and TCP port) for
+ * <b>router</b> into *<b>ap_out</b>. */
+void
+router_get_prim_orport(const routerinfo_t *router, tor_addr_port_t *ap_out)
+{
+ tor_assert(ap_out != NULL);
+ tor_addr_from_ipv4h(&ap_out->addr, router->addr);
+ ap_out->port = router->or_port;
+}
+
+/** Return 1 if any of <b>router</b>'s addresses are <b>addr</b>.
+ * Otherwise return 0. */
+int
+router_has_addr(const routerinfo_t *router, const tor_addr_t *addr)
+{
+ return
+ tor_addr_eq_ipv4h(addr, router->addr) ||
+ tor_addr_eq(&router->ipv6_addr, addr);
+}
+
+int
+router_has_orport(const routerinfo_t *router, const tor_addr_port_t *orport)
+{
+ return
+ (tor_addr_eq_ipv4h(&orport->addr, router->addr) &&
+ orport->port == router->or_port) ||
+ (tor_addr_eq(&orport->addr, &router->ipv6_addr) &&
+ orport->port == router->ipv6_orport);
+}
+
+/** Load the contents of <b>filename</b>, find the last line starting with
+ * <b>end_line</b>, ensure that its timestamp is not more than 25 hours in
+ * the past or more than 1 hour in the future with respect to <b>now</b>,
+ * and write the file contents starting with that line to *<b>out</b>.
+ * Return 1 for success, 0 if the file does not exist or is empty, or -1
+ * if the file does not contain a line matching these criteria or other
+ * failure. */
+static int
+load_stats_file(const char *filename, const char *end_line, time_t now,
+ char **out)
+{
+ int r = -1;
+ char *fname = get_datadir_fname(filename);
+ char *contents, *start = NULL, *tmp, timestr[ISO_TIME_LEN+1];
+ time_t written;
+ switch (file_status(fname)) {
+ case FN_FILE:
+ /* X022 Find an alternative to reading the whole file to memory. */
+ if ((contents = read_file_to_str(fname, 0, NULL))) {
+ tmp = strstr(contents, end_line);
+ /* Find last block starting with end_line */
+ while (tmp) {
+ start = tmp;
+ tmp = strstr(tmp + 1, end_line);
+ }
+ if (!start)
+ goto notfound;
+ if (strlen(start) < strlen(end_line) + 1 + sizeof(timestr))
+ goto notfound;
+ strlcpy(timestr, start + 1 + strlen(end_line), sizeof(timestr));
+ if (parse_iso_time(timestr, &written) < 0)
+ goto notfound;
+ if (written < now - (25*60*60) || written > now + (1*60*60))
+ goto notfound;
+ *out = tor_strdup(start);
+ r = 1;
+ }
+ notfound:
+ tor_free(contents);
+ break;
+ /* treat empty stats files as if the file doesn't exist */
+ case FN_NOENT:
+ case FN_EMPTY:
+ r = 0;
+ break;
+ case FN_ERROR:
+ case FN_DIR:
+ default:
+ break;
+ }
+ tor_free(fname);
+ return r;
+}
+
+/** Write the contents of <b>extrainfo</b> and aggregated statistics to
+ * *<b>s_out</b>, signing them with <b>ident_key</b>. Return 0 on
+ * success, negative on failure. */
+int
+extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
+ crypto_pk_t *ident_key,
+ const ed25519_keypair_t *signing_keypair)
+{
+ const or_options_t *options = get_options();
+ char identity[HEX_DIGEST_LEN+1];
+ char published[ISO_TIME_LEN+1];
+ char digest[DIGEST_LEN];
+ char *bandwidth_usage;
+ int result;
+ static int write_stats_to_extrainfo = 1;
+ char sig[DIROBJ_MAX_SIG_LEN+1];
+ char *s = NULL, *pre, *contents, *cp, *s_dup = NULL;
+ time_t now = time(NULL);
+ smartlist_t *chunks = smartlist_new();
+ extrainfo_t *ei_tmp = NULL;
+ const int emit_ed_sigs = signing_keypair &&
+ extrainfo->cache_info.signing_key_cert;
+ char *ed_cert_line = NULL;
+
+ base16_encode(identity, sizeof(identity),
+ extrainfo->cache_info.identity_digest, DIGEST_LEN);
+ format_iso_time(published, extrainfo->cache_info.published_on);
+ bandwidth_usage = rep_hist_get_bandwidth_lines();
+ if (emit_ed_sigs) {
+ if (!extrainfo->cache_info.signing_key_cert->signing_key_included ||
+ !ed25519_pubkey_eq(&extrainfo->cache_info.signing_key_cert->signed_key,
+ &signing_keypair->pubkey)) {
+ log_warn(LD_BUG, "Tried to sign a extrainfo descriptor with a "
+ "mismatched ed25519 key chain %d",
+ extrainfo->cache_info.signing_key_cert->signing_key_included);
+ goto err;
+ }
+ char ed_cert_base64[256];
+ if (base64_encode(ed_cert_base64, sizeof(ed_cert_base64),
+ (const char*)extrainfo->cache_info.signing_key_cert->encoded,
+ extrainfo->cache_info.signing_key_cert->encoded_len,
+ BASE64_ENCODE_MULTILINE) < 0) {
+ log_err(LD_BUG,"Couldn't base64-encode signing key certificate!");
+ goto err;
+ }
+ tor_asprintf(&ed_cert_line, "identity-ed25519\n"
+ "-----BEGIN ED25519 CERT-----\n"
+ "%s"
+ "-----END ED25519 CERT-----\n", ed_cert_base64);
+ } else {
+ ed_cert_line = tor_strdup("");
+ }
+
+ tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n%s",
+ extrainfo->nickname, identity,
+ ed_cert_line,
+ published, bandwidth_usage);
+ smartlist_add(chunks, pre);
+
+ if (geoip_is_loaded(AF_INET))
+ smartlist_add_asprintf(chunks, "geoip-db-digest %s\n",
+ geoip_db_digest(AF_INET));
+ if (geoip_is_loaded(AF_INET6))
+ smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n",
+ geoip_db_digest(AF_INET6));
+
+ if (options->ExtraInfoStatistics && write_stats_to_extrainfo) {
+ log_info(LD_GENERAL, "Adding stats to extra-info descriptor.");
+ if (options->DirReqStatistics &&
+ load_stats_file("stats"PATH_SEPARATOR"dirreq-stats",
+ "dirreq-stats-end", now, &contents) > 0) {
+ smartlist_add(chunks, contents);
+ }
+ if (options->HiddenServiceStatistics &&
+ load_stats_file("stats"PATH_SEPARATOR"hidserv-stats",
+ "hidserv-stats-end", now, &contents) > 0) {
+ smartlist_add(chunks, contents);
+ }
+ if (options->EntryStatistics &&
+ load_stats_file("stats"PATH_SEPARATOR"entry-stats",
+ "entry-stats-end", now, &contents) > 0) {
+ smartlist_add(chunks, contents);
+ }
+ if (options->CellStatistics &&
+ load_stats_file("stats"PATH_SEPARATOR"buffer-stats",
+ "cell-stats-end", now, &contents) > 0) {
+ smartlist_add(chunks, contents);
+ }
+ if (options->ExitPortStatistics &&
+ load_stats_file("stats"PATH_SEPARATOR"exit-stats",
+ "exit-stats-end", now, &contents) > 0) {
+ smartlist_add(chunks, contents);
+ }
+ if (options->ConnDirectionStatistics &&
+ load_stats_file("stats"PATH_SEPARATOR"conn-stats",
+ "conn-bi-direct", now, &contents) > 0) {
+ smartlist_add(chunks, contents);
+ }
+ }
+
+ if (options->PaddingStatistics) {
+ contents = rep_hist_get_padding_count_lines();
+ if (contents)
+ smartlist_add(chunks, contents);
+ }
+
+ /* Add information about the pluggable transports we support. */
+ if (options->ServerTransportPlugin) {
+ char *pluggable_transports = pt_get_extra_info_descriptor_string();
+ if (pluggable_transports)
+ smartlist_add(chunks, pluggable_transports);
+ }
+
+ if (should_record_bridge_info(options) && write_stats_to_extrainfo) {
+ const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
+ if (bridge_stats) {
+ smartlist_add_strdup(chunks, bridge_stats);
+ }
+ }
+
+ if (emit_ed_sigs) {
+ char sha256_digest[DIGEST256_LEN];
+ smartlist_add_strdup(chunks, "router-sig-ed25519 ");
+ crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
+ ED_DESC_SIGNATURE_PREFIX,
+ chunks, "", DIGEST_SHA256);
+ ed25519_signature_t ed_sig;
+ char buf[ED25519_SIG_BASE64_LEN+1];
+ if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN,
+ signing_keypair) < 0)
+ goto err;
+ if (ed25519_signature_to_base64(buf, &ed_sig) < 0)
+ goto err;
+
+ smartlist_add_asprintf(chunks, "%s\n", buf);
+ }
+
+ smartlist_add_strdup(chunks, "router-signature\n");
+ s = smartlist_join_strings(chunks, "", 0, NULL);
+
+ while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) {
+ /* So long as there are at least two chunks (one for the initial
+ * extra-info line and one for the router-signature), we can keep removing
+ * things. */
+ if (smartlist_len(chunks) > 2) {
+ /* We remove the next-to-last element (remember, len-1 is the last
+ element), since we need to keep the router-signature element. */
+ int idx = smartlist_len(chunks) - 2;
+ char *e = smartlist_get(chunks, idx);
+ smartlist_del_keeporder(chunks, idx);
+ log_warn(LD_GENERAL, "We just generated an extra-info descriptor "
+ "with statistics that exceeds the 50 KB "
+ "upload limit. Removing last added "
+ "statistics.");
+ tor_free(e);
+ tor_free(s);
+ s = smartlist_join_strings(chunks, "", 0, NULL);
+ } else {
+ log_warn(LD_BUG, "We just generated an extra-info descriptors that "
+ "exceeds the 50 KB upload limit.");
+ goto err;
+ }
+ }
+
+ memset(sig, 0, sizeof(sig));
+ if (router_get_extrainfo_hash(s, strlen(s), digest) < 0 ||
+ router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN,
+ ident_key) < 0) {
+ log_warn(LD_BUG, "Could not append signature to extra-info "
+ "descriptor.");
+ goto err;
+ }
+ smartlist_add_strdup(chunks, sig);
+ tor_free(s);
+ s = smartlist_join_strings(chunks, "", 0, NULL);
+
+ cp = s_dup = tor_strdup(s);
+ ei_tmp = extrainfo_parse_entry_from_string(cp, NULL, 1, NULL, NULL);
+ if (!ei_tmp) {
+ if (write_stats_to_extrainfo) {
+ log_warn(LD_GENERAL, "We just generated an extra-info descriptor "
+ "with statistics that we can't parse. Not "
+ "adding statistics to this or any future "
+ "extra-info descriptors.");
+ write_stats_to_extrainfo = 0;
+ result = extrainfo_dump_to_string(s_out, extrainfo, ident_key,
+ signing_keypair);
+ goto done;
+ } else {
+ log_warn(LD_BUG, "We just generated an extrainfo descriptor we "
+ "can't parse.");
+ goto err;
+ }
+ }
+
+ *s_out = s;
+ s = NULL; /* prevent free */
+ result = 0;
+ goto done;
+
+ err:
+ result = -1;
+
+ done:
+ tor_free(s);
+ SMARTLIST_FOREACH(chunks, char *, chunk, tor_free(chunk));
+ smartlist_free(chunks);
+ tor_free(s_dup);
+ tor_free(ed_cert_line);
+ extrainfo_free(ei_tmp);
+ tor_free(bandwidth_usage);
+
+ return result;
+}
+
+/** Return true iff <b>s</b> is a valid server nickname. (That is, a string
+ * containing between 1 and MAX_NICKNAME_LEN characters from
+ * LEGAL_NICKNAME_CHARACTERS.) */
+int
+is_legal_nickname(const char *s)
+{
+ size_t len;
+ tor_assert(s);
+ len = strlen(s);
+ return len > 0 && len <= MAX_NICKNAME_LEN &&
+ strspn(s,LEGAL_NICKNAME_CHARACTERS) == len;
+}
+
+/** Return true iff <b>s</b> is a valid server nickname or
+ * hex-encoded identity-key digest. */
+int
+is_legal_nickname_or_hexdigest(const char *s)
+{
+ if (*s!='$')
+ return is_legal_nickname(s);
+ else
+ return is_legal_hexdigest(s);
+}
+
+/** Return true iff <b>s</b> is a valid hex-encoded identity-key
+ * digest. (That is, an optional $, followed by 40 hex characters,
+ * followed by either nothing, or = or ~ followed by a nickname, or
+ * a character other than =, ~, or a hex character.)
+ */
+int
+is_legal_hexdigest(const char *s)
+{
+ size_t len;
+ tor_assert(s);
+ if (s[0] == '$') s++;
+ len = strlen(s);
+ if (len > HEX_DIGEST_LEN) {
+ if (s[HEX_DIGEST_LEN] == '=' ||
+ s[HEX_DIGEST_LEN] == '~') {
+ if (!is_legal_nickname(s+HEX_DIGEST_LEN+1))
+ return 0;
+ } else {
+ return 0;
+ }
+ }
+ return (len >= HEX_DIGEST_LEN &&
+ strspn(s,HEX_CHARACTERS)==HEX_DIGEST_LEN);
+}
+
+/**
+ * Longest allowed output of format_node_description, plus 1 character for
+ * NUL. This allows space for:
+ * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
+ * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
+ * plus a terminating NUL.
+ */
+#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN)
+
+/** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
+ * hold a human-readable description of a node with identity digest
+ * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>,
+ * and address <b>addr</b> or <b>addr32h</b>.
+ *
+ * The <b>nickname</b> and <b>addr</b> fields are optional and may be set to
+ * NULL. The <b>addr32h</b> field is optional and may be set to 0.
+ *
+ * Return a pointer to the front of <b>buf</b>.
+ */
+const char *
+format_node_description(char *buf,
+ const char *id_digest,
+ int is_named,
+ const char *nickname,
+ const tor_addr_t *addr,
+ uint32_t addr32h)
+{
+ char *cp;
+
+ if (!buf)
+ return "<NULL BUFFER>";
+
+ buf[0] = '$';
+ base16_encode(buf+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN);
+ cp = buf+1+HEX_DIGEST_LEN;
+ if (nickname) {
+ buf[1+HEX_DIGEST_LEN] = is_named ? '=' : '~';
+ strlcpy(buf+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1);
+ cp += strlen(cp);
+ }
+ if (addr32h || addr) {
+ memcpy(cp, " at ", 4);
+ cp += 4;
+ if (addr) {
+ tor_addr_to_str(cp, addr, TOR_ADDR_BUF_LEN, 0);
+ } else {
+ struct in_addr in;
+ in.s_addr = htonl(addr32h);
+ tor_inet_ntoa(&in, cp, INET_NTOA_BUF_LEN);
+ }
+ }
+ return buf;
+}
+
+/** Return a human-readable description of the routerinfo_t <b>ri</b>.
+ *
+ * This function is not thread-safe. Each call to this function invalidates
+ * previous values returned by this function.
+ */
+const char *
+router_describe(const routerinfo_t *ri)
+{
+ static char buf[NODE_DESC_BUF_LEN];
+
+ if (!ri)
+ return "<null>";
+ return format_node_description(buf,
+ ri->cache_info.identity_digest,
+ 0,
+ ri->nickname,
+ NULL,
+ ri->addr);
+}
+
+/** Return a human-readable description of the node_t <b>node</b>.
+ *
+ * This function is not thread-safe. Each call to this function invalidates
+ * previous values returned by this function.
+ */
+const char *
+node_describe(const node_t *node)
+{
+ static char buf[NODE_DESC_BUF_LEN];
+ const char *nickname = NULL;
+ uint32_t addr32h = 0;
+ int is_named = 0;
+
+ if (!node)
+ return "<null>";
+
+ if (node->rs) {
+ nickname = node->rs->nickname;
+ is_named = node->rs->is_named;
+ addr32h = node->rs->addr;
+ } else if (node->ri) {
+ nickname = node->ri->nickname;
+ addr32h = node->ri->addr;
+ }
+
+ return format_node_description(buf,
+ node->identity,
+ is_named,
+ nickname,
+ NULL,
+ addr32h);
+}
+
+/** Return a human-readable description of the routerstatus_t <b>rs</b>.
+ *
+ * This function is not thread-safe. Each call to this function invalidates
+ * previous values returned by this function.
+ */
+const char *
+routerstatus_describe(const routerstatus_t *rs)
+{
+ static char buf[NODE_DESC_BUF_LEN];
+
+ if (!rs)
+ return "<null>";
+ return format_node_description(buf,
+ rs->identity_digest,
+ rs->is_named,
+ rs->nickname,
+ NULL,
+ rs->addr);
+}
+
+/** Return a human-readable description of the extend_info_t <b>ei</b>.
+ *
+ * This function is not thread-safe. Each call to this function invalidates
+ * previous values returned by this function.
+ */
+const char *
+extend_info_describe(const extend_info_t *ei)
+{
+ static char buf[NODE_DESC_BUF_LEN];
+
+ if (!ei)
+ return "<null>";
+ return format_node_description(buf,
+ ei->identity_digest,
+ 0,
+ ei->nickname,
+ &ei->addr,
+ 0);
+}
+
+/** Set <b>buf</b> (which must have MAX_VERBOSE_NICKNAME_LEN+1 bytes) to the
+ * verbose representation of the identity of <b>router</b>. The format is:
+ * A dollar sign.
+ * The upper-case hexadecimal encoding of the SHA1 hash of router's identity.
+ * A "=" if the router is named (no longer implemented); a "~" if it is not.
+ * The router's nickname.
+ **/
+void
+router_get_verbose_nickname(char *buf, const routerinfo_t *router)
+{
+ buf[0] = '$';
+ base16_encode(buf+1, HEX_DIGEST_LEN+1, router->cache_info.identity_digest,
+ DIGEST_LEN);
+ buf[1+HEX_DIGEST_LEN] = '~';
+ strlcpy(buf+1+HEX_DIGEST_LEN+1, router->nickname, MAX_NICKNAME_LEN+1);
+}
+
+/** Forget that we have issued any router-related warnings, so that we'll
+ * warn again if we see the same errors. */
+void
+router_reset_warnings(void)
+{
+ if (warned_nonexistent_family) {
+ SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp));
+ smartlist_clear(warned_nonexistent_family);
+ }
+}
+
+/** Given a router purpose, convert it to a string. Don't call this on
+ * ROUTER_PURPOSE_UNKNOWN: The whole point of that value is that we don't
+ * know its string representation. */
+const char *
+router_purpose_to_string(uint8_t p)
+{
+ switch (p)
+ {
+ case ROUTER_PURPOSE_GENERAL: return "general";
+ case ROUTER_PURPOSE_BRIDGE: return "bridge";
+ case ROUTER_PURPOSE_CONTROLLER: return "controller";
+ default:
+ tor_assert(0);
+ }
+ return NULL;
+}
+
+/** Given a string, convert it to a router purpose. */
+uint8_t
+router_purpose_from_string(const char *s)
+{
+ if (!strcmp(s, "general"))
+ return ROUTER_PURPOSE_GENERAL;
+ else if (!strcmp(s, "bridge"))
+ return ROUTER_PURPOSE_BRIDGE;
+ else if (!strcmp(s, "controller"))
+ return ROUTER_PURPOSE_CONTROLLER;
+ else
+ return ROUTER_PURPOSE_UNKNOWN;
+}
+
+/** Release all static resources held in router.c */
+void
+router_free_all(void)
+{
+ crypto_pk_free(onionkey);
+ crypto_pk_free(lastonionkey);
+ crypto_pk_free(server_identitykey);
+ crypto_pk_free(client_identitykey);
+
+ tor_mutex_free(key_lock);
+ routerinfo_free(desc_routerinfo);
+ extrainfo_free(desc_extrainfo);
+ crypto_pk_free(authority_signing_key);
+ authority_cert_free(authority_key_certificate);
+ crypto_pk_free(legacy_signing_key);
+ authority_cert_free(legacy_key_certificate);
+
+ memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key));
+ memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
+
+ if (warned_nonexistent_family) {
+ SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp));
+ smartlist_free(warned_nonexistent_family);
+ }
+}
+
+/** Return a smartlist of tor_addr_port_t's with all the OR ports of
+ <b>ri</b>. Note that freeing of the items in the list as well as
+ the smartlist itself is the callers responsibility. */
+smartlist_t *
+router_get_all_orports(const routerinfo_t *ri)
+{
+ tor_assert(ri);
+ node_t fake_node;
+ memset(&fake_node, 0, sizeof(fake_node));
+ /* we don't modify ri, fake_node is passed as a const node_t *
+ */
+ fake_node.ri = (routerinfo_t *)ri;
+ return node_get_all_orports(&fake_node);
+}
+
+/* From the given RSA key object, convert it to ASN-1 encoded format and set
+ * the newly allocated object in onion_pkey_out. The length of the key is set
+ * in onion_pkey_len_out. */
+void
+router_set_rsa_onion_pkey(const crypto_pk_t *pk, char **onion_pkey_out,
+ size_t *onion_pkey_len_out)
+{
+ int len;
+ char buf[1024];
+
+ tor_assert(pk);
+ tor_assert(onion_pkey_out);
+ tor_assert(onion_pkey_len_out);
+
+ len = crypto_pk_asn1_encode(pk, buf, sizeof(buf));
+ if (BUG(len < 0)) {
+ goto done;
+ }
+
+ *onion_pkey_out = tor_memdup(buf, len);
+ *onion_pkey_len_out = len;
+
+ done:
+ return;
+}
+
+/* From an ASN-1 encoded onion pkey, return a newly allocated RSA key object.
+ * It is the caller responsability to free the returned object.
+ *
+ * Return NULL if the pkey is NULL, malformed or if the length is 0. */
+crypto_pk_t *
+router_get_rsa_onion_pkey(const char *pkey, size_t pkey_len)
+{
+ if (!pkey || pkey_len == 0) {
+ return NULL;
+ }
+ return crypto_pk_asn1_decode(pkey, pkey_len);
+}
diff --cc src/feature/stats/rephist.c
index 6bb680c5d,000000000..e24728ca7
mode 100644,000000..100644
--- a/src/feature/stats/rephist.c
+++ b/src/feature/stats/rephist.c
@@@ -1,3220 -1,0 +1,3220 @@@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rephist.c
+ * \brief Basic history and performance-tracking functionality.
+ *
+ * Basic history and performance-tracking functionality to remember
+ * which servers have worked in the past, how much bandwidth we've
+ * been using, which ports we tend to want, and so on; further,
+ * exit port statistics, cell statistics, and connection statistics.
+ *
+ * The history and information tracked in this module could sensibly be
+ * divided into several categories:
+ *
+ * <ul><li>Statistics used by authorities to remember the uptime and
+ * stability information about various relays, including "uptime",
+ * "weighted fractional uptime" and "mean time between failures".
+ *
+ * <li>Bandwidth usage history, used by relays to self-report how much
+ * bandwidth they've used for different purposes over last day or so,
+ * in order to generate the {dirreq-,}{read,write}-history lines in
+ * that they publish.
+ *
+ * <li>Predicted ports, used by clients to remember how long it's been
+ * since they opened an exit connection to each given target
+ * port. Clients use this information in order to try to keep circuits
+ * open to exit nodes that can connect to the ports that they care
+ * about. (The predicted ports mechanism also handles predicted circuit
+ * usage that _isn't_ port-specific, such as resolves, internal circuits,
+ * and so on.)
+ *
+ * <li>Public key operation counters, for tracking how many times we've
+ * done each public key operation. (This is unmaintained and we should
+ * remove it.)
+ *
+ * <li>Exit statistics by port, used by exits to keep track of the
+ * number of streams and bytes they've served at each exit port, so they
+ * can generate their exit-kibibytes-{read,written} and
+ * exit-streams-opened statistics.
+ *
+ * <li>Circuit stats, used by relays instances to tract circuit
+ * queue fullness and delay over time, and generate cell-processed-cells,
+ * cell-queued-cells, cell-time-in-queue, and cell-circuits-per-decile
+ * statistics.
+ *
+ * <li>Descriptor serving statistics, used by directory caches to track
+ * how many descriptors they've served.
+ *
+ * <li>Connection statistics, used by relays to track one-way and
+ * bidirectional connections.
+ *
+ * <li>Onion handshake statistics, used by relays to count how many
+ * TAP and ntor handshakes they've handled.
+ *
+ * <li>Hidden service statistics, used by relays to count rendezvous
+ * traffic and HSDir-stored descriptors.
+ *
+ * <li>Link protocol statistics, used by relays to count how many times
+ * each link protocol has been used.
+ *
+ * </ul>
+ *
+ * The entry points for this module are scattered throughout the
+ * codebase. Sending data, receiving data, connecting to a relay,
+ * losing a connection to a relay, and so on can all trigger a change in
+ * our current stats. Relays also invoke this module in order to
+ * extract their statistics when building routerinfo and extrainfo
+ * objects in router.c.
+ *
+ * TODO: This module should be broken up.
+ *
+ * (The "rephist" name originally stood for "reputation and history". )
+ **/
+
+#include "core/or/or.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "app/config/config.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/stats/rephist.h"
+#include "feature/relay/router.h"
+#include "feature/nodelist/routerlist.h"
+#include "ht.h"
+#include "core/or/channelpadding.h"
+#include "core/or/connection_or.h"
+#include "app/config/statefile.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/or_circuit_st.h"
+#include "app/config/or_state_st.h"
+
+#include "lib/container/bloomfilt.h"
+#include "lib/container/order.h"
+#include "lib/math/fp.h"
+#include "lib/math/laplace.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+static void bw_arrays_init(void);
+static void predicted_ports_alloc(void);
+
+/** Total number of bytes currently allocated in fields used by rephist.c. */
+uint64_t rephist_total_alloc=0;
+/** Number of or_history_t objects currently allocated. */
+uint32_t rephist_total_num=0;
+
+/** If the total weighted run count of all runs for a router ever falls
+ * below this amount, the router can be treated as having 0 MTBF. */
+#define STABILITY_EPSILON 0.0001
+/** Value by which to discount all old intervals for MTBF purposes. This
+ * is compounded every STABILITY_INTERVAL. */
+#define STABILITY_ALPHA 0.95
+/** Interval at which to discount all old intervals for MTBF purposes. */
+#define STABILITY_INTERVAL (12*60*60)
+/* (This combination of ALPHA, INTERVAL, and EPSILON makes it so that an
+ * interval that just ended counts twice as much as one that ended a week ago,
+ * 20X as much as one that ended a month ago, and routers that have had no
+ * uptime data for about half a year will get forgotten.) */
+
+/** History of an OR. */
+typedef struct or_history_t {
+ /** When did we start tracking this OR? */
+ time_t since;
+ /** When did we most recently note a change to this OR? */
+ time_t changed;
+
+ /** The address at which we most recently connected to this OR
+ * successfully. */
+ tor_addr_t last_reached_addr;
+
+ /** The port at which we most recently connected to this OR successfully */
+ uint16_t last_reached_port;
+
+ /* === For MTBF tracking: */
+ /** Weighted sum total of all times that this router has been online.
+ */
+ unsigned long weighted_run_length;
+ /** If the router is now online (according to stability-checking rules),
+ * when did it come online? */
+ time_t start_of_run;
+ /** Sum of weights for runs in weighted_run_length. */
+ double total_run_weights;
+ /* === For fractional uptime tracking: */
+ time_t start_of_downtime;
+ unsigned long weighted_uptime;
+ unsigned long total_weighted_time;
+} or_history_t;
+
+/**
+ * This structure holds accounting needed to calculate the padding overhead.
+ */
+typedef struct padding_counts_t {
+ /** Total number of cells we have received, including padding */
+ uint64_t read_cell_count;
+ /** Total number of cells we have sent, including padding */
+ uint64_t write_cell_count;
+ /** Total number of CELL_PADDING cells we have received */
+ uint64_t read_pad_cell_count;
+ /** Total number of CELL_PADDING cells we have sent */
+ uint64_t write_pad_cell_count;
+ /** Total number of read cells on padding-enabled conns */
+ uint64_t enabled_read_cell_count;
+ /** Total number of sent cells on padding-enabled conns */
+ uint64_t enabled_write_cell_count;
+ /** Total number of read CELL_PADDING cells on padding-enabled cons */
+ uint64_t enabled_read_pad_cell_count;
+ /** Total number of sent CELL_PADDING cells on padding-enabled cons */
+ uint64_t enabled_write_pad_cell_count;
+ /** Total number of RELAY_DROP cells we have received */
+ uint64_t read_drop_cell_count;
+ /** Total number of RELAY_DROP cells we have sent */
+ uint64_t write_drop_cell_count;
+ /** The maximum number of padding timers we've seen in 24 hours */
+ uint64_t maximum_chanpad_timers;
+ /** When did we first copy padding_current into padding_published? */
+ char first_published_at[ISO_TIME_LEN+1];
+} padding_counts_t;
+
+/** Holds the current values of our padding statistics.
+ * It is not published until it is transferred to padding_published. */
+static padding_counts_t padding_current;
+
+/** Remains fixed for a 24 hour period, and then is replaced
+ * by a redacted copy of padding_current */
+static padding_counts_t padding_published;
+
+/** When did we last multiply all routers' weighted_run_length and
+ * total_run_weights by STABILITY_ALPHA? */
+static time_t stability_last_downrated = 0;
+
+/** */
+static time_t started_tracking_stability = 0;
+
+/** Map from hex OR identity digest to or_history_t. */
+static digestmap_t *history_map = NULL;
+
+/** Return the or_history_t for the OR with identity digest <b>id</b>,
+ * creating it if necessary. */
+static or_history_t *
+get_or_history(const char* id)
+{
+ or_history_t *hist;
+
+ if (tor_digest_is_zero(id))
+ return NULL;
+
+ hist = digestmap_get(history_map, id);
+ if (!hist) {
+ hist = tor_malloc_zero(sizeof(or_history_t));
+ rephist_total_alloc += sizeof(or_history_t);
+ rephist_total_num++;
+ hist->since = hist->changed = time(NULL);
+ tor_addr_make_unspec(&hist->last_reached_addr);
+ digestmap_set(history_map, id, hist);
+ }
+ return hist;
+}
+
+/** Helper: free storage held by a single OR history entry. */
+static void
+free_or_history(void *_hist)
+{
+ or_history_t *hist = _hist;
+ rephist_total_alloc -= sizeof(or_history_t);
+ rephist_total_num--;
+ tor_free(hist);
+}
+
+/** Initialize the static data structures for tracking history. */
+void
+rep_hist_init(void)
+{
+ history_map = digestmap_new();
+ bw_arrays_init();
+ predicted_ports_alloc();
+}
+
+/** We have just decided that this router with identity digest <b>id</b> is
+ * reachable, meaning we will give it a "Running" flag for the next while. */
+void
+rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr,
+ const uint16_t at_port, time_t when)
+{
+ or_history_t *hist = get_or_history(id);
+ int was_in_run = 1;
+ char tbuf[ISO_TIME_LEN+1];
+ int addr_changed, port_changed;
+
+ tor_assert(hist);
+ tor_assert((!at_addr && !at_port) || (at_addr && at_port));
+
+ addr_changed = at_addr && !tor_addr_is_null(&hist->last_reached_addr) &&
+ tor_addr_compare(at_addr, &hist->last_reached_addr, CMP_EXACT) != 0;
+ port_changed = at_port && hist->last_reached_port &&
+ at_port != hist->last_reached_port;
+
+ if (!started_tracking_stability)
+ started_tracking_stability = time(NULL);
+ if (!hist->start_of_run) {
+ hist->start_of_run = when;
+ was_in_run = 0;
+ }
+ if (hist->start_of_downtime) {
+ long down_length;
+
+ format_local_iso_time(tbuf, hist->start_of_downtime);
+ log_info(LD_HIST, "Router %s is now Running; it had been down since %s.",
+ hex_str(id, DIGEST_LEN), tbuf);
+ if (was_in_run)
+ log_info(LD_HIST, " (Paradoxically, it was already Running too.)");
+
+ down_length = when - hist->start_of_downtime;
+ hist->total_weighted_time += down_length;
+ hist->start_of_downtime = 0;
+ } else if (addr_changed || port_changed) {
+ /* If we're reachable, but the address changed, treat this as some
+ * downtime. */
+ int penalty = get_options()->TestingTorNetwork ? 240 : 3600;
+ networkstatus_t *ns;
+
+ if ((ns = networkstatus_get_latest_consensus())) {
+ int fresh_interval = (int)(ns->fresh_until - ns->valid_after);
+ int live_interval = (int)(ns->valid_until - ns->valid_after);
+ /* on average, a descriptor addr change takes .5 intervals to make it
+ * into a consensus, and half a liveness period to make it to
+ * clients. */
+ penalty = (int)(fresh_interval + live_interval) / 2;
+ }
+ format_local_iso_time(tbuf, hist->start_of_run);
+ log_info(LD_HIST,"Router %s still seems Running, but its address appears "
+ "to have changed since the last time it was reachable. I'm "
+ "going to treat it as having been down for %d seconds",
+ hex_str(id, DIGEST_LEN), penalty);
+ rep_hist_note_router_unreachable(id, when-penalty);
+ rep_hist_note_router_reachable(id, NULL, 0, when);
+ } else {
+ format_local_iso_time(tbuf, hist->start_of_run);
+ if (was_in_run)
+ log_debug(LD_HIST, "Router %s is still Running; it has been Running "
+ "since %s", hex_str(id, DIGEST_LEN), tbuf);
+ else
+ log_info(LD_HIST,"Router %s is now Running; it was previously untracked",
+ hex_str(id, DIGEST_LEN));
+ }
+ if (at_addr)
+ tor_addr_copy(&hist->last_reached_addr, at_addr);
+ if (at_port)
+ hist->last_reached_port = at_port;
+}
+
+/** We have just decided that this router is unreachable, meaning
+ * we are taking away its "Running" flag. */
+void
+rep_hist_note_router_unreachable(const char *id, time_t when)
+{
+ or_history_t *hist = get_or_history(id);
+ char tbuf[ISO_TIME_LEN+1];
+ int was_running = 0;
+ if (!started_tracking_stability)
+ started_tracking_stability = time(NULL);
+
+ tor_assert(hist);
+ if (hist->start_of_run) {
+ /*XXXX We could treat failed connections differently from failed
+ * connect attempts. */
+ long run_length = when - hist->start_of_run;
+ format_local_iso_time(tbuf, hist->start_of_run);
+
+ hist->total_run_weights += 1.0;
+ hist->start_of_run = 0;
+ if (run_length < 0) {
+ unsigned long penalty = -run_length;
+#define SUBTRACT_CLAMPED(var, penalty) \
+ do { (var) = (var) < (penalty) ? 0 : (var) - (penalty); } while (0)
+
+ SUBTRACT_CLAMPED(hist->weighted_run_length, penalty);
+ SUBTRACT_CLAMPED(hist->weighted_uptime, penalty);
+ } else {
+ hist->weighted_run_length += run_length;
+ hist->weighted_uptime += run_length;
+ hist->total_weighted_time += run_length;
+ }
+ was_running = 1;
+ log_info(LD_HIST, "Router %s is now non-Running: it had previously been "
+ "Running since %s. Its total weighted uptime is %lu/%lu.",
+ hex_str(id, DIGEST_LEN), tbuf, hist->weighted_uptime,
+ hist->total_weighted_time);
+ }
+ if (!hist->start_of_downtime) {
+ hist->start_of_downtime = when;
+
+ if (!was_running)
+ log_info(LD_HIST, "Router %s is now non-Running; it was previously "
+ "untracked.", hex_str(id, DIGEST_LEN));
+ } else {
+ if (!was_running) {
+ format_local_iso_time(tbuf, hist->start_of_downtime);
+
+ log_info(LD_HIST, "Router %s is still non-Running; it has been "
+ "non-Running since %s.", hex_str(id, DIGEST_LEN), tbuf);
+ }
+ }
+}
+
+/** Mark a router with ID <b>id</b> as non-Running, and retroactively declare
+ * that it has never been running: give it no stability and no WFU. */
+void
+rep_hist_make_router_pessimal(const char *id, time_t when)
+{
+ or_history_t *hist = get_or_history(id);
+ tor_assert(hist);
+
+ rep_hist_note_router_unreachable(id, when);
+
+ hist->weighted_run_length = 0;
+ hist->weighted_uptime = 0;
+}
+
+/** Helper: Discount all old MTBF data, if it is time to do so. Return
+ * the time at which we should next discount MTBF data. */
+time_t
+rep_hist_downrate_old_runs(time_t now)
+{
+ digestmap_iter_t *orhist_it;
+ const char *digest1;
+ or_history_t *hist;
+ void *hist_p;
+ double alpha = 1.0;
+
+ if (!history_map)
+ history_map = digestmap_new();
+ if (!stability_last_downrated)
+ stability_last_downrated = now;
+ if (stability_last_downrated + STABILITY_INTERVAL > now)
+ return stability_last_downrated + STABILITY_INTERVAL;
+
+ /* Okay, we should downrate the data. By how much? */
+ while (stability_last_downrated + STABILITY_INTERVAL < now) {
+ stability_last_downrated += STABILITY_INTERVAL;
+ alpha *= STABILITY_ALPHA;
+ }
+
+ log_info(LD_HIST, "Discounting all old stability info by a factor of %f",
+ alpha);
+
+ /* Multiply every w_r_l, t_r_w pair by alpha. */
+ for (orhist_it = digestmap_iter_init(history_map);
+ !digestmap_iter_done(orhist_it);
+ orhist_it = digestmap_iter_next(history_map,orhist_it)) {
+ digestmap_iter_get(orhist_it, &digest1, &hist_p);
+ hist = hist_p;
+
+ hist->weighted_run_length =
+ (unsigned long)(hist->weighted_run_length * alpha);
+ hist->total_run_weights *= alpha;
+
+ hist->weighted_uptime = (unsigned long)(hist->weighted_uptime * alpha);
+ hist->total_weighted_time = (unsigned long)
+ (hist->total_weighted_time * alpha);
+ }
+
+ return stability_last_downrated + STABILITY_INTERVAL;
+}
+
+/** Helper: Return the weighted MTBF of the router with history <b>hist</b>. */
+static double
+get_stability(or_history_t *hist, time_t when)
+{
+ long total = hist->weighted_run_length;
+ double total_weights = hist->total_run_weights;
+
+ if (hist->start_of_run) {
+ /* We're currently in a run. Let total and total_weights hold the values
+ * they would hold if the current run were to end now. */
+ total += (when-hist->start_of_run);
+ total_weights += 1.0;
+ }
+ if (total_weights < STABILITY_EPSILON) {
+ /* Round down to zero, and avoid divide-by-zero. */
+ return 0.0;
+ }
+
+ return total / total_weights;
+}
+
+/** Return the total amount of time we've been observing, with each run of
+ * time downrated by the appropriate factor. */
+static long
+get_total_weighted_time(or_history_t *hist, time_t when)
+{
+ long total = hist->total_weighted_time;
+ if (hist->start_of_run) {
+ total += (when - hist->start_of_run);
+ } else if (hist->start_of_downtime) {
+ total += (when - hist->start_of_downtime);
+ }
+ return total;
+}
+
+/** Helper: Return the weighted percent-of-time-online of the router with
+ * history <b>hist</b>. */
+static double
+get_weighted_fractional_uptime(or_history_t *hist, time_t when)
+{
+ long total = hist->total_weighted_time;
+ long up = hist->weighted_uptime;
+
+ if (hist->start_of_run) {
+ long run_length = (when - hist->start_of_run);
+ up += run_length;
+ total += run_length;
+ } else if (hist->start_of_downtime) {
+ total += (when - hist->start_of_downtime);
+ }
+
+ if (!total) {
+ /* Avoid calling anybody's uptime infinity (which should be impossible if
+ * the code is working), or NaN (which can happen for any router we haven't
+ * observed up or down yet). */
+ return 0.0;
+ }
+
+ return ((double) up) / total;
+}
+
+/** Return how long the router whose identity digest is <b>id</b> has
+ * been reachable. Return 0 if the router is unknown or currently deemed
+ * unreachable. */
+long
+rep_hist_get_uptime(const char *id, time_t when)
+{
+ or_history_t *hist = get_or_history(id);
+ if (!hist)
+ return 0;
+ if (!hist->start_of_run || when < hist->start_of_run)
+ return 0;
+ return when - hist->start_of_run;
+}
+
+/** Return an estimated MTBF for the router whose identity digest is
+ * <b>id</b>. Return 0 if the router is unknown. */
+double
+rep_hist_get_stability(const char *id, time_t when)
+{
+ or_history_t *hist = get_or_history(id);
+ if (!hist)
+ return 0.0;
+
+ return get_stability(hist, when);
+}
+
+/** Return an estimated percent-of-time-online for the router whose identity
+ * digest is <b>id</b>. Return 0 if the router is unknown. */
+double
+rep_hist_get_weighted_fractional_uptime(const char *id, time_t when)
+{
+ or_history_t *hist = get_or_history(id);
+ if (!hist)
+ return 0.0;
+
+ return get_weighted_fractional_uptime(hist, when);
+}
+
+/** Return a number representing how long we've known about the router whose
+ * digest is <b>id</b>. Return 0 if the router is unknown.
+ *
+ * Be careful: this measure increases monotonically as we know the router for
+ * longer and longer, but it doesn't increase linearly.
+ */
+long
+rep_hist_get_weighted_time_known(const char *id, time_t when)
+{
+ or_history_t *hist = get_or_history(id);
+ if (!hist)
+ return 0;
+
+ return get_total_weighted_time(hist, when);
+}
+
+/** Return true if we've been measuring MTBFs for long enough to
+ * pronounce on Stability. */
+int
+rep_hist_have_measured_enough_stability(void)
+{
+ /* XXXX++ This doesn't do so well when we change our opinion
+ * as to whether we're tracking router stability. */
+ return started_tracking_stability < time(NULL) - 4*60*60;
+}
+
+/** Log all the reliability data we have remembered, with the chosen
+ * severity.
+ */
+void
+rep_hist_dump_stats(time_t now, int severity)
+{
+ digestmap_iter_t *orhist_it;
+ const char *name1, *digest1;
+ char hexdigest1[HEX_DIGEST_LEN+1];
+ or_history_t *or_history;
+ void *or_history_p;
+ const node_t *node;
+
+ rep_history_clean(now - get_options()->RephistTrackTime);
+
+ tor_log(severity, LD_HIST, "--------------- Dumping history information:");
+
+ for (orhist_it = digestmap_iter_init(history_map);
+ !digestmap_iter_done(orhist_it);
+ orhist_it = digestmap_iter_next(history_map,orhist_it)) {
+ double s;
+ long stability;
+ digestmap_iter_get(orhist_it, &digest1, &or_history_p);
+ or_history = (or_history_t*) or_history_p;
+
+ if ((node = node_get_by_id(digest1)) && node_get_nickname(node))
+ name1 = node_get_nickname(node);
+ else
+ name1 = "(unknown)";
+ base16_encode(hexdigest1, sizeof(hexdigest1), digest1, DIGEST_LEN);
+ s = get_stability(or_history, now);
+ stability = (long)s;
+ tor_log(severity, LD_HIST,
+ "OR %s [%s]: wmtbf %lu:%02lu:%02lu",
+ name1, hexdigest1,
+ stability/3600, (stability/60)%60, stability%60);
+ }
+}
+
+/** Remove history info for routers/links that haven't changed since
+ * <b>before</b>.
+ */
+void
+rep_history_clean(time_t before)
+{
+ int authority = authdir_mode(get_options());
+ or_history_t *or_history;
+ void *or_history_p;
+ digestmap_iter_t *orhist_it;
+ const char *d1;
+
+ orhist_it = digestmap_iter_init(history_map);
+ while (!digestmap_iter_done(orhist_it)) {
+ int should_remove;
+ digestmap_iter_get(orhist_it, &d1, &or_history_p);
+ or_history = or_history_p;
+
+ should_remove = authority ?
+ (or_history->total_run_weights < STABILITY_EPSILON &&
+ !or_history->start_of_run)
+ : (or_history->changed < before);
+ if (should_remove) {
+ orhist_it = digestmap_iter_next_rmv(history_map, orhist_it);
+ free_or_history(or_history);
+ continue;
+ }
+ orhist_it = digestmap_iter_next(history_map, orhist_it);
+ }
+}
+
+/** Write MTBF data to disk. Return 0 on success, negative on failure.
+ *
+ * If <b>missing_means_down</b>, then if we're about to write an entry
+ * that is still considered up but isn't in our routerlist, consider it
+ * to be down. */
+int
+rep_hist_record_mtbf_data(time_t now, int missing_means_down)
+{
+ char time_buf[ISO_TIME_LEN+1];
+
+ digestmap_iter_t *orhist_it;
+ const char *digest;
+ void *or_history_p;
+ or_history_t *hist;
+ open_file_t *open_file = NULL;
+ FILE *f;
+
+ {
+ char *filename = get_datadir_fname("router-stability");
+ f = start_writing_to_stdio_file(filename, OPEN_FLAGS_REPLACE|O_TEXT, 0600,
+ &open_file);
+ tor_free(filename);
+ if (!f)
+ return -1;
+ }
+
+ /* File format is:
+ * FormatLine *KeywordLine Data
+ *
+ * FormatLine = "format 1" NL
+ * KeywordLine = Keyword SP Arguments NL
+ * Data = "data" NL *RouterMTBFLine "." NL
+ * RouterMTBFLine = Fingerprint SP WeightedRunLen SP
+ * TotalRunWeights [SP S=StartRunTime] NL
+ */
+#define PUT(s) STMT_BEGIN if (fputs((s),f)<0) goto err; STMT_END
+#define PRINTF(args) STMT_BEGIN if (fprintf args <0) goto err; STMT_END
+
+ PUT("format 2\n");
+
+ format_iso_time(time_buf, time(NULL));
+ PRINTF((f, "stored-at %s\n", time_buf));
+
+ if (started_tracking_stability) {
+ format_iso_time(time_buf, started_tracking_stability);
+ PRINTF((f, "tracked-since %s\n", time_buf));
+ }
+ if (stability_last_downrated) {
+ format_iso_time(time_buf, stability_last_downrated);
+ PRINTF((f, "last-downrated %s\n", time_buf));
+ }
+
+ PUT("data\n");
+
+ /* XXX Nick: now bridge auths record this for all routers too.
+ * Should we make them record it only for bridge routers? -RD
+ * Not for 0.2.0. -NM */
+ for (orhist_it = digestmap_iter_init(history_map);
+ !digestmap_iter_done(orhist_it);
+ orhist_it = digestmap_iter_next(history_map,orhist_it)) {
+ char dbuf[HEX_DIGEST_LEN+1];
+ const char *t = NULL;
+ digestmap_iter_get(orhist_it, &digest, &or_history_p);
+ hist = (or_history_t*) or_history_p;
+
+ base16_encode(dbuf, sizeof(dbuf), digest, DIGEST_LEN);
+
+ if (missing_means_down && hist->start_of_run &&
+ !connection_or_digest_is_known_relay(digest)) {
+ /* We think this relay is running, but it's not listed in our
+ * consensus. Somehow it fell out without telling us it went
+ * down. Complain and also correct it. */
+ log_info(LD_HIST,
+ "Relay '%s' is listed as up in rephist, but it's not in "
+ "our routerlist. Correcting.", dbuf);
+ rep_hist_note_router_unreachable(digest, now);
+ }
+
+ PRINTF((f, "R %s\n", dbuf));
+ if (hist->start_of_run > 0) {
+ format_iso_time(time_buf, hist->start_of_run);
+ t = time_buf;
+ }
+ PRINTF((f, "+MTBF %lu %.5f%s%s\n",
+ hist->weighted_run_length, hist->total_run_weights,
+ t ? " S=" : "", t ? t : ""));
+ t = NULL;
+ if (hist->start_of_downtime > 0) {
+ format_iso_time(time_buf, hist->start_of_downtime);
+ t = time_buf;
+ }
+ PRINTF((f, "+WFU %lu %lu%s%s\n",
+ hist->weighted_uptime, hist->total_weighted_time,
+ t ? " S=" : "", t ? t : ""));
+ }
+
+ PUT(".\n");
+
+#undef PUT
+#undef PRINTF
+
+ return finish_writing_to_file(open_file);
+ err:
+ abort_writing_to_file(open_file);
+ return -1;
+}
+
+/** Helper: return the first j >= i such that !strcmpstart(sl[j], prefix) and
+ * such that no line sl[k] with i <= k < j starts with "R ". Return -1 if no
+ * such line exists. */
+static int
+find_next_with(smartlist_t *sl, int i, const char *prefix)
+{
+ for ( ; i < smartlist_len(sl); ++i) {
+ const char *line = smartlist_get(sl, i);
+ if (!strcmpstart(line, prefix))
+ return i;
+ if (!strcmpstart(line, "R "))
+ return -1;
+ }
+ return -1;
+}
+
+/** How many bad times has parse_possibly_bad_iso_time() parsed? */
+static int n_bogus_times = 0;
+/** Parse the ISO-formatted time in <b>s</b> into *<b>time_out</b>, but
+ * round any pre-1970 date to Jan 1, 1970. */
+static int
+parse_possibly_bad_iso_time(const char *s, time_t *time_out)
+{
+ int year;
+ char b[5];
+ strlcpy(b, s, sizeof(b));
+ b[4] = '\0';
+ year = (int)tor_parse_long(b, 10, 0, INT_MAX, NULL, NULL);
+ if (year < 1970) {
+ *time_out = 0;
+ ++n_bogus_times;
+ return 0;
+ } else
+ return parse_iso_time(s, time_out);
+}
+
+/** We've read a time <b>t</b> from a file stored at <b>stored_at</b>, which
+ * says we started measuring at <b>started_measuring</b>. Return a new number
+ * that's about as much before <b>now</b> as <b>t</b> was before
+ * <b>stored_at</b>.
+ */
+static inline time_t
+correct_time(time_t t, time_t now, time_t stored_at, time_t started_measuring)
+{
+ if (t < started_measuring - 24*60*60*365)
+ return 0;
+ else if (t < started_measuring)
+ return started_measuring;
+ else if (t > stored_at)
+ return 0;
+ else {
+ long run_length = stored_at - t;
+ t = (time_t)(now - run_length);
+ if (t < started_measuring)
+ t = started_measuring;
+ return t;
+ }
+}
+
+/** Load MTBF data from disk. Returns 0 on success or recoverable error, -1
+ * on failure. */
+int
+rep_hist_load_mtbf_data(time_t now)
+{
+ /* XXXX won't handle being called while history is already populated. */
+ smartlist_t *lines;
+ const char *line = NULL;
+ int r=0, i;
+ time_t last_downrated = 0, stored_at = 0, tracked_since = 0;
+ time_t latest_possible_start = now;
+ long format = -1;
+
+ {
+ char *filename = get_datadir_fname("router-stability");
+ char *d = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+ tor_free(filename);
+ if (!d)
+ return -1;
+ lines = smartlist_new();
+ smartlist_split_string(lines, d, "\n", SPLIT_SKIP_SPACE, 0);
+ tor_free(d);
+ }
+
+ {
+ const char *firstline;
+ if (smartlist_len(lines)>4) {
+ firstline = smartlist_get(lines, 0);
+ if (!strcmpstart(firstline, "format "))
+ format = tor_parse_long(firstline+strlen("format "),
+ 10, -1, LONG_MAX, NULL, NULL);
+ }
+ }
+ if (format != 1 && format != 2) {
+ log_warn(LD_HIST,
+ "Unrecognized format in mtbf history file. Skipping.");
+ goto err;
+ }
+ for (i = 1; i < smartlist_len(lines); ++i) {
+ line = smartlist_get(lines, i);
+ if (!strcmp(line, "data"))
+ break;
+ if (!strcmpstart(line, "last-downrated ")) {
+ if (parse_iso_time(line+strlen("last-downrated "), &last_downrated)<0)
+ log_warn(LD_HIST,"Couldn't parse downrate time in mtbf "
+ "history file.");
+ }
+ if (!strcmpstart(line, "stored-at ")) {
+ if (parse_iso_time(line+strlen("stored-at "), &stored_at)<0)
+ log_warn(LD_HIST,"Couldn't parse stored time in mtbf "
+ "history file.");
+ }
+ if (!strcmpstart(line, "tracked-since ")) {
+ if (parse_iso_time(line+strlen("tracked-since "), &tracked_since)<0)
+ log_warn(LD_HIST,"Couldn't parse started-tracking time in mtbf "
+ "history file.");
+ }
+ }
+ if (last_downrated > now)
+ last_downrated = now;
+ if (tracked_since > now)
+ tracked_since = now;
+
+ if (!stored_at) {
+ log_warn(LD_HIST, "No stored time recorded.");
+ goto err;
+ }
+
+ if (line && !strcmp(line, "data"))
+ ++i;
+
+ n_bogus_times = 0;
+
+ for (; i < smartlist_len(lines); ++i) {
+ char digest[DIGEST_LEN];
+ char hexbuf[HEX_DIGEST_LEN+1];
+ char mtbf_timebuf[ISO_TIME_LEN+1];
+ char wfu_timebuf[ISO_TIME_LEN+1];
+ time_t start_of_run = 0;
+ time_t start_of_downtime = 0;
+ int have_mtbf = 0, have_wfu = 0;
+ long wrl = 0;
+ double trw = 0;
+ long wt_uptime = 0, total_wt_time = 0;
+ int n;
+ or_history_t *hist;
+ line = smartlist_get(lines, i);
+ if (!strcmp(line, "."))
+ break;
+
+ mtbf_timebuf[0] = '\0';
+ wfu_timebuf[0] = '\0';
+
+ if (format == 1) {
+ n = tor_sscanf(line, "%40s %ld %lf S=%10s %8s",
+ hexbuf, &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11);
+ if (n != 3 && n != 5) {
+ log_warn(LD_HIST, "Couldn't scan line %s", escaped(line));
+ continue;
+ }
+ have_mtbf = 1;
+ } else {
+ // format == 2.
+ int mtbf_idx, wfu_idx;
+ if (strcmpstart(line, "R ") || strlen(line) < 2+HEX_DIGEST_LEN)
+ continue;
+ strlcpy(hexbuf, line+2, sizeof(hexbuf));
+ mtbf_idx = find_next_with(lines, i+1, "+MTBF ");
+ wfu_idx = find_next_with(lines, i+1, "+WFU ");
+ if (mtbf_idx >= 0) {
+ const char *mtbfline = smartlist_get(lines, mtbf_idx);
+ n = tor_sscanf(mtbfline, "+MTBF %lu %lf S=%10s %8s",
+ &wrl, &trw, mtbf_timebuf, mtbf_timebuf+11);
+ if (n == 2 || n == 4) {
+ have_mtbf = 1;
+ } else {
+ log_warn(LD_HIST, "Couldn't scan +MTBF line %s",
+ escaped(mtbfline));
+ }
+ }
+ if (wfu_idx >= 0) {
+ const char *wfuline = smartlist_get(lines, wfu_idx);
+ n = tor_sscanf(wfuline, "+WFU %lu %lu S=%10s %8s",
+ &wt_uptime, &total_wt_time,
+ wfu_timebuf, wfu_timebuf+11);
+ if (n == 2 || n == 4) {
+ have_wfu = 1;
+ } else {
+ log_warn(LD_HIST, "Couldn't scan +WFU line %s", escaped(wfuline));
+ }
+ }
+ if (wfu_idx > i)
+ i = wfu_idx;
+ if (mtbf_idx > i)
+ i = mtbf_idx;
+ }
+ if (base16_decode(digest, DIGEST_LEN,
+ hexbuf, HEX_DIGEST_LEN) != DIGEST_LEN) {
+ log_warn(LD_HIST, "Couldn't hex string %s", escaped(hexbuf));
+ continue;
+ }
+ hist = get_or_history(digest);
+ if (!hist)
+ continue;
+
+ if (have_mtbf) {
+ if (mtbf_timebuf[0]) {
+ mtbf_timebuf[10] = ' ';
+ if (parse_possibly_bad_iso_time(mtbf_timebuf, &start_of_run)<0)
+ log_warn(LD_HIST, "Couldn't parse time %s",
+ escaped(mtbf_timebuf));
+ }
+ hist->start_of_run = correct_time(start_of_run, now, stored_at,
+ tracked_since);
+ if (hist->start_of_run < latest_possible_start + wrl)
+ latest_possible_start = (time_t)(hist->start_of_run - wrl);
+
+ hist->weighted_run_length = wrl;
+ hist->total_run_weights = trw;
+ }
+ if (have_wfu) {
+ if (wfu_timebuf[0]) {
+ wfu_timebuf[10] = ' ';
+ if (parse_possibly_bad_iso_time(wfu_timebuf, &start_of_downtime)<0)
+ log_warn(LD_HIST, "Couldn't parse time %s", escaped(wfu_timebuf));
+ }
+ }
+ hist->start_of_downtime = correct_time(start_of_downtime, now, stored_at,
+ tracked_since);
+ hist->weighted_uptime = wt_uptime;
+ hist->total_weighted_time = total_wt_time;
+ }
+ if (strcmp(line, "."))
+ log_warn(LD_HIST, "Truncated MTBF file.");
+
+ if (tracked_since < 86400*365) /* Recover from insanely early value. */
+ tracked_since = latest_possible_start;
+
+ stability_last_downrated = last_downrated;
+ started_tracking_stability = tracked_since;
+
+ goto done;
+ err:
+ r = -1;
+ done:
+ SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+ smartlist_free(lines);
+ return r;
+}
+
+/** For how many seconds do we keep track of individual per-second bandwidth
+ * totals? */
+#define NUM_SECS_ROLLING_MEASURE 10
+/** How large are the intervals for which we track and report bandwidth use? */
+#define NUM_SECS_BW_SUM_INTERVAL (24*60*60)
+/** How far in the past do we remember and publish bandwidth use? */
+#define NUM_SECS_BW_SUM_IS_VALID (5*24*60*60)
+/** How many bandwidth usage intervals do we remember? (derived) */
+#define NUM_TOTALS (NUM_SECS_BW_SUM_IS_VALID/NUM_SECS_BW_SUM_INTERVAL)
+
+/** Structure to track bandwidth use, and remember the maxima for a given
+ * time period.
+ */
+typedef struct bw_array_t {
+ /** Observation array: Total number of bytes transferred in each of the last
+ * NUM_SECS_ROLLING_MEASURE seconds. This is used as a circular array. */
+ uint64_t obs[NUM_SECS_ROLLING_MEASURE];
+ int cur_obs_idx; /**< Current position in obs. */
+ time_t cur_obs_time; /**< Time represented in obs[cur_obs_idx] */
+ uint64_t total_obs; /**< Total for all members of obs except
+ * obs[cur_obs_idx] */
+ uint64_t max_total; /**< Largest value that total_obs has taken on in the
+ * current period. */
+ uint64_t total_in_period; /**< Total bytes transferred in the current
+ * period. */
+
+ /** When does the next period begin? */
+ time_t next_period;
+ /** Where in 'maxima' should the maximum bandwidth usage for the current
+ * period be stored? */
+ int next_max_idx;
+ /** How many values in maxima/totals have been set ever? */
+ int num_maxes_set;
+ /** Circular array of the maximum
+ * bandwidth-per-NUM_SECS_ROLLING_MEASURE usage for the last
+ * NUM_TOTALS periods */
+ uint64_t maxima[NUM_TOTALS];
+ /** Circular array of the total bandwidth usage for the last NUM_TOTALS
+ * periods */
+ uint64_t totals[NUM_TOTALS];
+} bw_array_t;
+
+/** Shift the current period of b forward by one. */
+static void
+commit_max(bw_array_t *b)
+{
+ /* Store total from current period. */
+ b->totals[b->next_max_idx] = b->total_in_period;
+ /* Store maximum from current period. */
+ b->maxima[b->next_max_idx++] = b->max_total;
+ /* Advance next_period and next_max_idx */
+ b->next_period += NUM_SECS_BW_SUM_INTERVAL;
+ if (b->next_max_idx == NUM_TOTALS)
+ b->next_max_idx = 0;
+ if (b->num_maxes_set < NUM_TOTALS)
+ ++b->num_maxes_set;
+ /* Reset max_total. */
+ b->max_total = 0;
+ /* Reset total_in_period. */
+ b->total_in_period = 0;
+}
+
+/** Shift the current observation time of <b>b</b> forward by one second. */
+static inline void
+advance_obs(bw_array_t *b)
+{
+ int nextidx;
+ uint64_t total;
+
+ /* Calculate the total bandwidth for the last NUM_SECS_ROLLING_MEASURE
+ * seconds; adjust max_total as needed.*/
+ total = b->total_obs + b->obs[b->cur_obs_idx];
+ if (total > b->max_total)
+ b->max_total = total;
+
+ nextidx = b->cur_obs_idx+1;
+ if (nextidx == NUM_SECS_ROLLING_MEASURE)
+ nextidx = 0;
+
+ b->total_obs = total - b->obs[nextidx];
+ b->obs[nextidx]=0;
+ b->cur_obs_idx = nextidx;
+
+ if (++b->cur_obs_time >= b->next_period)
+ commit_max(b);
+}
+
+/** Add <b>n</b> bytes to the number of bytes in <b>b</b> for second
+ * <b>when</b>. */
+static inline void
+add_obs(bw_array_t *b, time_t when, uint64_t n)
+{
+ if (when < b->cur_obs_time)
+ return; /* Don't record data in the past. */
+
+ /* If we're currently adding observations for an earlier second than
+ * 'when', advance b->cur_obs_time and b->cur_obs_idx by an
+ * appropriate number of seconds, and do all the other housekeeping. */
+ while (when > b->cur_obs_time) {
+ /* Doing this one second at a time is potentially inefficient, if we start
+ with a state file that is very old. Fortunately, it doesn't seem to
+ show up in profiles, so we can just ignore it for now. */
+ advance_obs(b);
+ }
+
+ b->obs[b->cur_obs_idx] += n;
+ b->total_in_period += n;
+}
+
+/** Allocate, initialize, and return a new bw_array. */
+static bw_array_t *
+bw_array_new(void)
+{
+ bw_array_t *b;
+ time_t start;
+ b = tor_malloc_zero(sizeof(bw_array_t));
+ rephist_total_alloc += sizeof(bw_array_t);
+ start = time(NULL);
+ b->cur_obs_time = start;
+ b->next_period = start + NUM_SECS_BW_SUM_INTERVAL;
+ return b;
+}
+
+#define bw_array_free(val) \
+ FREE_AND_NULL(bw_array_t, bw_array_free_, (val))
+
+/** Free storage held by bandwidth array <b>b</b>. */
+static void
+bw_array_free_(bw_array_t *b)
+{
+ if (!b) {
+ return;
+ }
+
+ rephist_total_alloc -= sizeof(bw_array_t);
+ tor_free(b);
+}
+
+/** Recent history of bandwidth observations for read operations. */
+static bw_array_t *read_array = NULL;
+/** Recent history of bandwidth observations for write operations. */
+static bw_array_t *write_array = NULL;
+/** Recent history of bandwidth observations for read operations for the
+ directory protocol. */
+static bw_array_t *dir_read_array = NULL;
+/** Recent history of bandwidth observations for write operations for the
+ directory protocol. */
+static bw_array_t *dir_write_array = NULL;
+
+/** Set up [dir-]read_array and [dir-]write_array, freeing them if they
+ * already exist. */
+static void
+bw_arrays_init(void)
+{
+ bw_array_free(read_array);
+ bw_array_free(write_array);
+ bw_array_free(dir_read_array);
+ bw_array_free(dir_write_array);
+
+ read_array = bw_array_new();
+ write_array = bw_array_new();
+ dir_read_array = bw_array_new();
+ dir_write_array = bw_array_new();
+}
+
+/** Remember that we read <b>num_bytes</b> bytes in second <b>when</b>.
+ *
+ * Add num_bytes to the current running total for <b>when</b>.
+ *
+ * <b>when</b> can go back to time, but it's safe to ignore calls
+ * earlier than the latest <b>when</b> you've heard of.
+ */
+void
+rep_hist_note_bytes_written(size_t num_bytes, time_t when)
+{
+/* Maybe a circular array for recent seconds, and step to a new point
+ * every time a new second shows up. Or simpler is to just to have
+ * a normal array and push down each item every second; it's short.
+ */
+/* When a new second has rolled over, compute the sum of the bytes we've
+ * seen over when-1 to when-1-NUM_SECS_ROLLING_MEASURE, and stick it
+ * somewhere. See rep_hist_bandwidth_assess() below.
+ */
+ add_obs(write_array, when, num_bytes);
+}
+
+/** Remember that we wrote <b>num_bytes</b> bytes in second <b>when</b>.
+ * (like rep_hist_note_bytes_written() above)
+ */
+void
+rep_hist_note_bytes_read(size_t num_bytes, time_t when)
+{
+/* if we're smart, we can make this func and the one above share code */
+ add_obs(read_array, when, num_bytes);
+}
+
+/** Remember that we wrote <b>num_bytes</b> directory bytes in second
+ * <b>when</b>. (like rep_hist_note_bytes_written() above)
+ */
+void
+rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when)
+{
+ add_obs(dir_write_array, when, num_bytes);
+}
+
+/** Remember that we read <b>num_bytes</b> directory bytes in second
+ * <b>when</b>. (like rep_hist_note_bytes_written() above)
+ */
+void
+rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when)
+{
+ add_obs(dir_read_array, when, num_bytes);
+}
+
+/** Helper: Return the largest value in b->maxima. (This is equal to the
+ * most bandwidth used in any NUM_SECS_ROLLING_MEASURE period for the last
+ * NUM_SECS_BW_SUM_IS_VALID seconds.)
+ */
+static uint64_t
+find_largest_max(bw_array_t *b)
+{
+ int i;
+ uint64_t max;
+ max=0;
+ for (i=0; i<NUM_TOTALS; ++i) {
+ if (b->maxima[i]>max)
+ max = b->maxima[i];
+ }
+ return max;
+}
+
+/** Find the largest sums in the past NUM_SECS_BW_SUM_IS_VALID (roughly)
+ * seconds. Find one sum for reading and one for writing. They don't have
+ * to be at the same time.
+ *
+ * Return the smaller of these sums, divided by NUM_SECS_ROLLING_MEASURE.
+ */
- int
- rep_hist_bandwidth_assess(void)
++MOCK_IMPL(int,
++rep_hist_bandwidth_assess,(void))
+{
+ uint64_t w,r;
+ r = find_largest_max(read_array);
+ w = find_largest_max(write_array);
+ if (r>w)
+ return (int)(((double)w)/NUM_SECS_ROLLING_MEASURE);
+ else
+ return (int)(((double)r)/NUM_SECS_ROLLING_MEASURE);
+}
+
+/** Print the bandwidth history of b (either [dir-]read_array or
+ * [dir-]write_array) into the buffer pointed to by buf. The format is
+ * simply comma separated numbers, from oldest to newest.
+ *
+ * It returns the number of bytes written.
+ */
+static size_t
+rep_hist_fill_bandwidth_history(char *buf, size_t len, const bw_array_t *b)
+{
+ char *cp = buf;
+ int i, n;
+ const or_options_t *options = get_options();
+ uint64_t cutoff;
+
+ if (b->num_maxes_set <= b->next_max_idx) {
+ /* We haven't been through the circular array yet; time starts at i=0.*/
+ i = 0;
+ } else {
+ /* We've been around the array at least once. The next i to be
+ overwritten is the oldest. */
+ i = b->next_max_idx;
+ }
+
+ if (options->RelayBandwidthRate) {
+ /* We don't want to report that we used more bandwidth than the max we're
+ * willing to relay; otherwise everybody will know how much traffic
+ * we used ourself. */
+ cutoff = options->RelayBandwidthRate * NUM_SECS_BW_SUM_INTERVAL;
+ } else {
+ cutoff = UINT64_MAX;
+ }
+
+ for (n=0; n<b->num_maxes_set; ++n,++i) {
+ uint64_t total;
+ if (i >= NUM_TOTALS)
+ i -= NUM_TOTALS;
+ tor_assert(i < NUM_TOTALS);
+ /* Round the bandwidth used down to the nearest 1k. */
+ total = b->totals[i] & ~0x3ff;
+ if (total > cutoff)
+ total = cutoff;
+
+ if (n==(b->num_maxes_set-1))
+ tor_snprintf(cp, len-(cp-buf), "%"PRIu64, (total));
+ else
+ tor_snprintf(cp, len-(cp-buf), "%"PRIu64",", (total));
+ cp += strlen(cp);
+ }
+ return cp-buf;
+}
+
+/** Allocate and return lines for representing this server's bandwidth
+ * history in its descriptor. We publish these lines in our extra-info
+ * descriptor.
+ */
+char *
+rep_hist_get_bandwidth_lines(void)
+{
+ char *buf, *cp;
+ char t[ISO_TIME_LEN+1];
+ int r;
+ bw_array_t *b = NULL;
+ const char *desc = NULL;
+ size_t len;
+
+ /* [dirreq-](read|write)-history yyyy-mm-dd HH:MM:SS (n s) n,n,n... */
+/* The n,n,n part above. Largest representation of a uint64_t is 20 chars
+ * long, plus the comma. */
+#define MAX_HIST_VALUE_LEN (21*NUM_TOTALS)
+ len = (67+MAX_HIST_VALUE_LEN)*4;
+ buf = tor_malloc_zero(len);
+ cp = buf;
+ for (r=0;r<4;++r) {
+ char tmp[MAX_HIST_VALUE_LEN];
+ size_t slen;
+ switch (r) {
+ case 0:
+ b = write_array;
+ desc = "write-history";
+ break;
+ case 1:
+ b = read_array;
+ desc = "read-history";
+ break;
+ case 2:
+ b = dir_write_array;
+ desc = "dirreq-write-history";
+ break;
+ case 3:
+ b = dir_read_array;
+ desc = "dirreq-read-history";
+ break;
+ }
+ tor_assert(b);
+ slen = rep_hist_fill_bandwidth_history(tmp, MAX_HIST_VALUE_LEN, b);
+ /* If we don't have anything to write, skip to the next entry. */
+ if (slen == 0)
+ continue;
+ format_iso_time(t, b->next_period-NUM_SECS_BW_SUM_INTERVAL);
+ tor_snprintf(cp, len-(cp-buf), "%s %s (%d s) ",
+ desc, t, NUM_SECS_BW_SUM_INTERVAL);
+ cp += strlen(cp);
+ strlcat(cp, tmp, len-(cp-buf));
+ cp += slen;
+ strlcat(cp, "\n", len-(cp-buf));
+ ++cp;
+ }
+ return buf;
+}
+
+/** Write a single bw_array_t into the Values, Ends, Interval, and Maximum
+ * entries of an or_state_t. Done before writing out a new state file. */
+static void
+rep_hist_update_bwhist_state_section(or_state_t *state,
+ const bw_array_t *b,
+ smartlist_t **s_values,
+ smartlist_t **s_maxima,
+ time_t *s_begins,
+ int *s_interval)
+{
+ int i,j;
+ uint64_t maxval;
+
+ if (*s_values) {
+ SMARTLIST_FOREACH(*s_values, char *, val, tor_free(val));
+ smartlist_free(*s_values);
+ }
+ if (*s_maxima) {
+ SMARTLIST_FOREACH(*s_maxima, char *, val, tor_free(val));
+ smartlist_free(*s_maxima);
+ }
+ if (! server_mode(get_options())) {
+ /* Clients don't need to store bandwidth history persistently;
+ * force these values to the defaults. */
+ /* FFFF we should pull the default out of config.c's state table,
+ * so we don't have two defaults. */
+ if (*s_begins != 0 || *s_interval != 900) {
+ time_t now = time(NULL);
+ time_t save_at = get_options()->AvoidDiskWrites ? now+3600 : now+600;
+ or_state_mark_dirty(state, save_at);
+ }
+ *s_begins = 0;
+ *s_interval = 900;
+ *s_values = smartlist_new();
+ *s_maxima = smartlist_new();
+ return;
+ }
+ *s_begins = b->next_period;
+ *s_interval = NUM_SECS_BW_SUM_INTERVAL;
+
+ *s_values = smartlist_new();
+ *s_maxima = smartlist_new();
+ /* Set i to first position in circular array */
+ i = (b->num_maxes_set <= b->next_max_idx) ? 0 : b->next_max_idx;
+ for (j=0; j < b->num_maxes_set; ++j,++i) {
+ if (i >= NUM_TOTALS)
+ i = 0;
+ smartlist_add_asprintf(*s_values, "%"PRIu64,
+ (b->totals[i] & ~0x3ff));
+ maxval = b->maxima[i] / NUM_SECS_ROLLING_MEASURE;
+ smartlist_add_asprintf(*s_maxima, "%"PRIu64,
+ (maxval & ~0x3ff));
+ }
+ smartlist_add_asprintf(*s_values, "%"PRIu64,
+ (b->total_in_period & ~0x3ff));
+ maxval = b->max_total / NUM_SECS_ROLLING_MEASURE;
+ smartlist_add_asprintf(*s_maxima, "%"PRIu64,
+ (maxval & ~0x3ff));
+}
+
+/** Update <b>state</b> with the newest bandwidth history. Done before
+ * writing out a new state file. */
+void
+rep_hist_update_state(or_state_t *state)
+{
+#define UPDATE(arrname,st) \
+ rep_hist_update_bwhist_state_section(state,\
+ (arrname),\
+ &state->BWHistory ## st ## Values, \
+ &state->BWHistory ## st ## Maxima, \
+ &state->BWHistory ## st ## Ends, \
+ &state->BWHistory ## st ## Interval)
+
+ UPDATE(write_array, Write);
+ UPDATE(read_array, Read);
+ UPDATE(dir_write_array, DirWrite);
+ UPDATE(dir_read_array, DirRead);
+
+ if (server_mode(get_options())) {
+ or_state_mark_dirty(state, time(NULL)+(2*3600));
+ }
+#undef UPDATE
+}
+
+/** Load a single bw_array_t from its Values, Ends, Maxima, and Interval
+ * entries in an or_state_t. Done while reading the state file. */
+static int
+rep_hist_load_bwhist_state_section(bw_array_t *b,
+ const smartlist_t *s_values,
+ const smartlist_t *s_maxima,
+ const time_t s_begins,
+ const int s_interval)
+{
+ time_t now = time(NULL);
+ int retval = 0;
+ time_t start;
+
+ uint64_t v, mv;
+ int i,ok,ok_m = 0;
+ int have_maxima = s_maxima && s_values &&
+ (smartlist_len(s_values) == smartlist_len(s_maxima));
+
+ if (s_values && s_begins >= now - NUM_SECS_BW_SUM_INTERVAL*NUM_TOTALS) {
+ start = s_begins - s_interval*(smartlist_len(s_values));
+ if (start > now)
+ return 0;
+ b->cur_obs_time = start;
+ b->next_period = start + NUM_SECS_BW_SUM_INTERVAL;
+ SMARTLIST_FOREACH_BEGIN(s_values, const char *, cp) {
+ const char *maxstr = NULL;
+ v = tor_parse_uint64(cp, 10, 0, UINT64_MAX, &ok, NULL);
+ if (have_maxima) {
+ maxstr = smartlist_get(s_maxima, cp_sl_idx);
+ mv = tor_parse_uint64(maxstr, 10, 0, UINT64_MAX, &ok_m, NULL);
+ mv *= NUM_SECS_ROLLING_MEASURE;
+ } else {
+ /* No maxima known; guess average rate to be conservative. */
+ mv = (v / s_interval) * NUM_SECS_ROLLING_MEASURE;
+ }
+ if (!ok) {
+ retval = -1;
+ log_notice(LD_HIST, "Could not parse value '%s' into a number.'",cp);
+ }
+ if (maxstr && !ok_m) {
+ retval = -1;
+ log_notice(LD_HIST, "Could not parse maximum '%s' into a number.'",
+ maxstr);
+ }
+
+ if (start < now) {
+ time_t cur_start = start;
+ time_t actual_interval_len = s_interval;
+ uint64_t cur_val = 0;
+ /* Calculate the average per second. This is the best we can do
+ * because our state file doesn't have per-second resolution. */
+ if (start + s_interval > now)
+ actual_interval_len = now - start;
+ cur_val = v / actual_interval_len;
+ /* This is potentially inefficient, but since we don't do it very
+ * often it should be ok. */
+ while (cur_start < start + actual_interval_len) {
+ add_obs(b, cur_start, cur_val);
+ ++cur_start;
+ }
+ b->max_total = mv;
+ /* This will result in some fairly choppy history if s_interval
+ * is not the same as NUM_SECS_BW_SUM_INTERVAL. XXXX */
+ start += actual_interval_len;
+ }
+ } SMARTLIST_FOREACH_END(cp);
+ }
+
+ /* Clean up maxima and observed */
+ for (i=0; i<NUM_SECS_ROLLING_MEASURE; ++i) {
+ b->obs[i] = 0;
+ }
+ b->total_obs = 0;
+
+ return retval;
+}
+
+/** Set bandwidth history from the state file we just loaded. */
+int
+rep_hist_load_state(or_state_t *state, char **err)
+{
+ int all_ok = 1;
+
+ /* Assert they already have been malloced */
+ tor_assert(read_array && write_array);
+ tor_assert(dir_read_array && dir_write_array);
+
+#define LOAD(arrname,st) \
+ if (rep_hist_load_bwhist_state_section( \
+ (arrname), \
+ state->BWHistory ## st ## Values, \
+ state->BWHistory ## st ## Maxima, \
+ state->BWHistory ## st ## Ends, \
+ state->BWHistory ## st ## Interval)<0) \
+ all_ok = 0
+
+ LOAD(write_array, Write);
+ LOAD(read_array, Read);
+ LOAD(dir_write_array, DirWrite);
+ LOAD(dir_read_array, DirRead);
+
+#undef LOAD
+ if (!all_ok) {
+ *err = tor_strdup("Parsing of bandwidth history values failed");
+ /* and create fresh arrays */
+ bw_arrays_init();
+ return -1;
+ }
+ return 0;
+}
+
+/*********************************************************************/
+
+/** A single predicted port: used to remember which ports we've made
+ * connections to, so that we can try to keep making circuits that can handle
+ * those ports. */
+typedef struct predicted_port_t {
+ /** The port we connected to */
+ uint16_t port;
+ /** The time at which we last used it */
+ time_t time;
+} predicted_port_t;
+
+/** A list of port numbers that have been used recently. */
+static smartlist_t *predicted_ports_list=NULL;
+/** How long do we keep predicting circuits? */
+static int prediction_timeout=0;
+/** When was the last time we added a prediction entry (HS or port) */
+static time_t last_prediction_add_time=0;
+
+/**
+ * How much time left until we stop predicting circuits?
+ */
+int
+predicted_ports_prediction_time_remaining(time_t now)
+{
+ time_t idle_delta;
+
+ /* Protect against overflow of return value. This can happen if the clock
+ * jumps backwards in time. Update the last prediction time (aka last
+ * active time) to prevent it. This update is preferable to using monotonic
+ * time because it prevents clock jumps into the past from simply causing
+ * very long idle timeouts while the monotonic time stands still. */
+ if (last_prediction_add_time > now) {
+ last_prediction_add_time = now;
+ idle_delta = 0;
+ } else {
+ idle_delta = now - last_prediction_add_time;
+ }
+
+ /* Protect against underflow of the return value. This can happen for very
+ * large periods of inactivity/system sleep. */
+ if (idle_delta > prediction_timeout)
+ return 0;
+
+ if (BUG((prediction_timeout - idle_delta) > INT_MAX)) {
+ return INT_MAX;
+ }
+
+ return (int)(prediction_timeout - idle_delta);
+}
+
+/** We just got an application request for a connection with
+ * port <b>port</b>. Remember it for the future, so we can keep
+ * some circuits open that will exit to this port.
+ */
+static void
+add_predicted_port(time_t now, uint16_t port)
+{
+ predicted_port_t *pp = tor_malloc(sizeof(predicted_port_t));
+
+ // If the list is empty, re-randomize predicted ports lifetime
+ if (!any_predicted_circuits(now)) {
+ prediction_timeout = channelpadding_get_circuits_available_timeout();
+ }
+
+ last_prediction_add_time = now;
+
+ log_info(LD_CIRC,
+ "New port prediction added. Will continue predictive circ building "
+ "for %d more seconds.",
+ predicted_ports_prediction_time_remaining(now));
+
+ pp->port = port;
+ pp->time = now;
+ rephist_total_alloc += sizeof(*pp);
+ smartlist_add(predicted_ports_list, pp);
+}
+
+/**
+ * Allocate whatever memory and structs are needed for predicting
+ * which ports will be used. Also seed it with port 80, so we'll build
+ * circuits on start-up.
+ */
+static void
+predicted_ports_alloc(void)
+{
+ predicted_ports_list = smartlist_new();
+}
+
+void
+predicted_ports_init(void)
+{
+ add_predicted_port(time(NULL), 443); // Add a port to get us started
+}
+
+/** Free whatever memory is needed for predicting which ports will
+ * be used.
+ */
+static void
+predicted_ports_free_all(void)
+{
+ rephist_total_alloc -=
+ smartlist_len(predicted_ports_list)*sizeof(predicted_port_t);
+ SMARTLIST_FOREACH(predicted_ports_list, predicted_port_t *,
+ pp, tor_free(pp));
+ smartlist_free(predicted_ports_list);
+}
+
+/** Remember that <b>port</b> has been asked for as of time <b>now</b>.
+ * This is used for predicting what sorts of streams we'll make in the
+ * future and making exit circuits to anticipate that.
+ */
+void
+rep_hist_note_used_port(time_t now, uint16_t port)
+{
+ tor_assert(predicted_ports_list);
+
+ if (!port) /* record nothing */
+ return;
+
+ SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
+ if (pp->port == port) {
+ pp->time = now;
+
+ last_prediction_add_time = now;
+ log_info(LD_CIRC,
+ "New port prediction added. Will continue predictive circ "
+ "building for %d more seconds.",
+ predicted_ports_prediction_time_remaining(now));
+ return;
+ }
+ } SMARTLIST_FOREACH_END(pp);
+ /* it's not there yet; we need to add it */
+ add_predicted_port(now, port);
+}
+
+/** Return a newly allocated pointer to a list of uint16_t * for ports that
+ * are likely to be asked for in the near future.
+ */
+smartlist_t *
+rep_hist_get_predicted_ports(time_t now)
+{
+ int predicted_circs_relevance_time;
+ smartlist_t *out = smartlist_new();
+ tor_assert(predicted_ports_list);
+
+ predicted_circs_relevance_time = prediction_timeout;
+
+ /* clean out obsolete entries */
+ SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
+ if (pp->time + predicted_circs_relevance_time < now) {
+ log_debug(LD_CIRC, "Expiring predicted port %d", pp->port);
+
+ rephist_total_alloc -= sizeof(predicted_port_t);
+ tor_free(pp);
+ SMARTLIST_DEL_CURRENT(predicted_ports_list, pp);
+ } else {
+ smartlist_add(out, tor_memdup(&pp->port, sizeof(uint16_t)));
+ }
+ } SMARTLIST_FOREACH_END(pp);
+ return out;
+}
+
+/**
+ * Take a list of uint16_t *, and remove every port in the list from the
+ * current list of predicted ports.
+ */
+void
+rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports)
+{
+ /* Let's do this on O(N), not O(N^2). */
+ bitarray_t *remove_ports = bitarray_init_zero(UINT16_MAX);
+ SMARTLIST_FOREACH(rmv_ports, const uint16_t *, p,
+ bitarray_set(remove_ports, *p));
+ SMARTLIST_FOREACH_BEGIN(predicted_ports_list, predicted_port_t *, pp) {
+ if (bitarray_is_set(remove_ports, pp->port)) {
+ tor_free(pp);
+ rephist_total_alloc -= sizeof(*pp);
+ SMARTLIST_DEL_CURRENT(predicted_ports_list, pp);
+ }
+ } SMARTLIST_FOREACH_END(pp);
+ bitarray_free(remove_ports);
+}
+
+/** The user asked us to do a resolve. Rather than keeping track of
+ * timings and such of resolves, we fake it for now by treating
+ * it the same way as a connection to port 80. This way we will continue
+ * to have circuits lying around if the user only uses Tor for resolves.
+ */
+void
+rep_hist_note_used_resolve(time_t now)
+{
+ rep_hist_note_used_port(now, 80);
+}
+
+/** The last time at which we needed an internal circ. */
+static time_t predicted_internal_time = 0;
+/** The last time we needed an internal circ with good uptime. */
+static time_t predicted_internal_uptime_time = 0;
+/** The last time we needed an internal circ with good capacity. */
+static time_t predicted_internal_capacity_time = 0;
+
+/** Remember that we used an internal circ at time <b>now</b>. */
+void
+rep_hist_note_used_internal(time_t now, int need_uptime, int need_capacity)
+{
+ // If the list is empty, re-randomize predicted ports lifetime
+ if (!any_predicted_circuits(now)) {
+ prediction_timeout = channelpadding_get_circuits_available_timeout();
+ }
+
+ last_prediction_add_time = now;
+
+ log_info(LD_CIRC,
+ "New port prediction added. Will continue predictive circ building "
+ "for %d more seconds.",
+ predicted_ports_prediction_time_remaining(now));
+
+ predicted_internal_time = now;
+ if (need_uptime)
+ predicted_internal_uptime_time = now;
+ if (need_capacity)
+ predicted_internal_capacity_time = now;
+}
+
+/** Return 1 if we've used an internal circ recently; else return 0. */
+int
+rep_hist_get_predicted_internal(time_t now, int *need_uptime,
+ int *need_capacity)
+{
+ int predicted_circs_relevance_time;
+
+ predicted_circs_relevance_time = prediction_timeout;
+
+ if (!predicted_internal_time) { /* initialize it */
+ predicted_internal_time = now;
+ predicted_internal_uptime_time = now;
+ predicted_internal_capacity_time = now;
+ }
+ if (predicted_internal_time + predicted_circs_relevance_time < now)
+ return 0; /* too long ago */
+ if (predicted_internal_uptime_time + predicted_circs_relevance_time >= now)
+ *need_uptime = 1;
+ // Always predict that we need capacity.
+ *need_capacity = 1;
+ return 1;
+}
+
+/** Any ports used lately? These are pre-seeded if we just started
+ * up or if we're running a hidden service. */
+int
+any_predicted_circuits(time_t now)
+{
+ int predicted_circs_relevance_time;
+ predicted_circs_relevance_time = prediction_timeout;
+
+ return smartlist_len(predicted_ports_list) ||
+ predicted_internal_time + predicted_circs_relevance_time >= now;
+}
+
+/** Return 1 if we have no need for circuits currently, else return 0. */
+int
+rep_hist_circbuilding_dormant(time_t now)
+{
+ const or_options_t *options = get_options();
+
+ if (any_predicted_circuits(now))
+ return 0;
+
+ /* see if we'll still need to build testing circuits */
+ if (server_mode(options) &&
+ (!check_whether_orport_reachable(options) ||
+ !circuit_enough_testing_circs()))
+ return 0;
+ if (!check_whether_dirport_reachable(options))
+ return 0;
+
+ return 1;
+}
+
+/*** Exit port statistics ***/
+
+/* Some constants */
+/** To what multiple should byte numbers be rounded up? */
+#define EXIT_STATS_ROUND_UP_BYTES 1024
+/** To what multiple should stream counts be rounded up? */
+#define EXIT_STATS_ROUND_UP_STREAMS 4
+/** Number of TCP ports */
+#define EXIT_STATS_NUM_PORTS 65536
+/** Top n ports that will be included in exit stats. */
+#define EXIT_STATS_TOP_N_PORTS 10
+
+/* The following data structures are arrays and no fancy smartlists or maps,
+ * so that all write operations can be done in constant time. This comes at
+ * the price of some memory (1.25 MB) and linear complexity when writing
+ * stats for measuring relays. */
+/** Number of bytes read in current period by exit port */
+static uint64_t *exit_bytes_read = NULL;
+/** Number of bytes written in current period by exit port */
+static uint64_t *exit_bytes_written = NULL;
+/** Number of streams opened in current period by exit port */
+static uint32_t *exit_streams = NULL;
+
+/** Start time of exit stats or 0 if we're not collecting exit stats. */
+static time_t start_of_exit_stats_interval;
+
+/** Initialize exit port stats. */
+void
+rep_hist_exit_stats_init(time_t now)
+{
+ start_of_exit_stats_interval = now;
+ exit_bytes_read = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t));
+ exit_bytes_written = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint64_t));
+ exit_streams = tor_calloc(EXIT_STATS_NUM_PORTS, sizeof(uint32_t));
+}
+
+/** Reset counters for exit port statistics. */
+void
+rep_hist_reset_exit_stats(time_t now)
+{
+ start_of_exit_stats_interval = now;
+ memset(exit_bytes_read, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t));
+ memset(exit_bytes_written, 0, EXIT_STATS_NUM_PORTS * sizeof(uint64_t));
+ memset(exit_streams, 0, EXIT_STATS_NUM_PORTS * sizeof(uint32_t));
+}
+
+/** Stop collecting exit port stats in a way that we can re-start doing
+ * so in rep_hist_exit_stats_init(). */
+void
+rep_hist_exit_stats_term(void)
+{
+ start_of_exit_stats_interval = 0;
+ tor_free(exit_bytes_read);
+ tor_free(exit_bytes_written);
+ tor_free(exit_streams);
+}
+
+/** Helper for qsort: compare two ints. Does not handle overflow properly,
+ * but works fine for sorting an array of port numbers, which is what we use
+ * it for. */
+static int
+compare_int_(const void *x, const void *y)
+{
+ return (*(int*)x - *(int*)y);
+}
+
+/** Return a newly allocated string containing the exit port statistics
+ * until <b>now</b>, or NULL if we're not collecting exit stats. Caller
+ * must ensure start_of_exit_stats_interval is in the past. */
+char *
+rep_hist_format_exit_stats(time_t now)
+{
+ int i, j, top_elements = 0, cur_min_idx = 0, cur_port;
+ uint64_t top_bytes[EXIT_STATS_TOP_N_PORTS];
+ int top_ports[EXIT_STATS_TOP_N_PORTS];
+ uint64_t cur_bytes = 0, other_read = 0, other_written = 0,
+ total_read = 0, total_written = 0;
+ uint32_t total_streams = 0, other_streams = 0;
+ smartlist_t *written_strings, *read_strings, *streams_strings;
+ char *written_string, *read_string, *streams_string;
+ char t[ISO_TIME_LEN+1];
+ char *result;
+
+ if (!start_of_exit_stats_interval)
+ return NULL; /* Not initialized. */
+
+ tor_assert(now >= start_of_exit_stats_interval);
+
+ /* Go through all ports to find the n ports that saw most written and
+ * read bytes.
+ *
+ * Invariant: at the end of the loop for iteration i,
+ * total_read is the sum of all exit_bytes_read[0..i]
+ * total_written is the sum of all exit_bytes_written[0..i]
+ * total_stream is the sum of all exit_streams[0..i]
+ *
+ * top_elements = MAX(EXIT_STATS_TOP_N_PORTS,
+ * #{j | 0 <= j <= i && volume(i) > 0})
+ *
+ * For all 0 <= j < top_elements,
+ * top_bytes[j] > 0
+ * 0 <= top_ports[j] <= 65535
+ * top_bytes[j] = volume(top_ports[j])
+ *
+ * There is no j in 0..i and k in 0..top_elements such that:
+ * volume(j) > top_bytes[k] AND j is not in top_ports[0..top_elements]
+ *
+ * There is no j!=cur_min_idx in 0..top_elements such that:
+ * top_bytes[j] < top_bytes[cur_min_idx]
+ *
+ * where volume(x) == exit_bytes_read[x]+exit_bytes_written[x]
+ *
+ * Worst case: O(EXIT_STATS_NUM_PORTS * EXIT_STATS_TOP_N_PORTS)
+ */
+ for (i = 1; i < EXIT_STATS_NUM_PORTS; i++) {
+ total_read += exit_bytes_read[i];
+ total_written += exit_bytes_written[i];
+ total_streams += exit_streams[i];
+ cur_bytes = exit_bytes_read[i] + exit_bytes_written[i];
+ if (cur_bytes == 0) {
+ continue;
+ }
+ if (top_elements < EXIT_STATS_TOP_N_PORTS) {
+ top_bytes[top_elements] = cur_bytes;
+ top_ports[top_elements++] = i;
+ } else if (cur_bytes > top_bytes[cur_min_idx]) {
+ top_bytes[cur_min_idx] = cur_bytes;
+ top_ports[cur_min_idx] = i;
+ } else {
+ continue;
+ }
+ cur_min_idx = 0;
+ for (j = 1; j < top_elements; j++) {
+ if (top_bytes[j] < top_bytes[cur_min_idx]) {
+ cur_min_idx = j;
+ }
+ }
+ }
+
+ /* Add observations of top ports to smartlists. */
+ written_strings = smartlist_new();
+ read_strings = smartlist_new();
+ streams_strings = smartlist_new();
+ other_read = total_read;
+ other_written = total_written;
+ other_streams = total_streams;
+ /* Sort the ports; this puts them out of sync with top_bytes, but we
+ * won't be using top_bytes again anyway */
+ qsort(top_ports, top_elements, sizeof(int), compare_int_);
+ for (j = 0; j < top_elements; j++) {
+ cur_port = top_ports[j];
+ if (exit_bytes_written[cur_port] > 0) {
+ uint64_t num = round_uint64_to_next_multiple_of(
+ exit_bytes_written[cur_port],
+ EXIT_STATS_ROUND_UP_BYTES);
+ num /= 1024;
+ smartlist_add_asprintf(written_strings, "%d=%"PRIu64,
+ cur_port, (num));
+ other_written -= exit_bytes_written[cur_port];
+ }
+ if (exit_bytes_read[cur_port] > 0) {
+ uint64_t num = round_uint64_to_next_multiple_of(
+ exit_bytes_read[cur_port],
+ EXIT_STATS_ROUND_UP_BYTES);
+ num /= 1024;
+ smartlist_add_asprintf(read_strings, "%d=%"PRIu64,
+ cur_port, (num));
+ other_read -= exit_bytes_read[cur_port];
+ }
+ if (exit_streams[cur_port] > 0) {
+ uint32_t num = round_uint32_to_next_multiple_of(
+ exit_streams[cur_port],
+ EXIT_STATS_ROUND_UP_STREAMS);
+ smartlist_add_asprintf(streams_strings, "%d=%u", cur_port, num);
+ other_streams -= exit_streams[cur_port];
+ }
+ }
+
+ /* Add observations of other ports in a single element. */
+ other_written = round_uint64_to_next_multiple_of(other_written,
+ EXIT_STATS_ROUND_UP_BYTES);
+ other_written /= 1024;
+ smartlist_add_asprintf(written_strings, "other=%"PRIu64,
+ (other_written));
+ other_read = round_uint64_to_next_multiple_of(other_read,
+ EXIT_STATS_ROUND_UP_BYTES);
+ other_read /= 1024;
+ smartlist_add_asprintf(read_strings, "other=%"PRIu64,
+ (other_read));
+ other_streams = round_uint32_to_next_multiple_of(other_streams,
+ EXIT_STATS_ROUND_UP_STREAMS);
+ smartlist_add_asprintf(streams_strings, "other=%u", other_streams);
+
+ /* Join all observations in single strings. */
+ written_string = smartlist_join_strings(written_strings, ",", 0, NULL);
+ read_string = smartlist_join_strings(read_strings, ",", 0, NULL);
+ streams_string = smartlist_join_strings(streams_strings, ",", 0, NULL);
+ SMARTLIST_FOREACH(written_strings, char *, cp, tor_free(cp));
+ SMARTLIST_FOREACH(read_strings, char *, cp, tor_free(cp));
+ SMARTLIST_FOREACH(streams_strings, char *, cp, tor_free(cp));
+ smartlist_free(written_strings);
+ smartlist_free(read_strings);
+ smartlist_free(streams_strings);
+
+ /* Put everything together. */
+ format_iso_time(t, now);
+ tor_asprintf(&result, "exit-stats-end %s (%d s)\n"
+ "exit-kibibytes-written %s\n"
+ "exit-kibibytes-read %s\n"
+ "exit-streams-opened %s\n",
+ t, (unsigned) (now - start_of_exit_stats_interval),
+ written_string,
+ read_string,
+ streams_string);
+ tor_free(written_string);
+ tor_free(read_string);
+ tor_free(streams_string);
+ return result;
+}
+
+/** If 24 hours have passed since the beginning of the current exit port
+ * stats period, write exit stats to $DATADIR/stats/exit-stats (possibly
+ * overwriting an existing file) and reset counters. Return when we would
+ * next want to write exit stats or 0 if we never want to write. */
+time_t
+rep_hist_exit_stats_write(time_t now)
+{
+ char *str = NULL;
+
+ if (!start_of_exit_stats_interval)
+ return 0; /* Not initialized. */
+ if (start_of_exit_stats_interval + WRITE_STATS_INTERVAL > now)
+ goto done; /* Not ready to write. */
+
+ log_info(LD_HIST, "Writing exit port statistics to disk.");
+
+ /* Generate history string. */
+ str = rep_hist_format_exit_stats(now);
+
+ /* Reset counters. */
+ rep_hist_reset_exit_stats(now);
+
+ /* Try to write to disk. */
+ if (!check_or_create_data_subdir("stats")) {
+ write_to_data_subdir("stats", "exit-stats", str, "exit port statistics");
+ }
+
+ done:
+ tor_free(str);
+ return start_of_exit_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/** Note that we wrote <b>num_written</b> bytes and read <b>num_read</b>
+ * bytes to/from an exit connection to <b>port</b>. */
+void
+rep_hist_note_exit_bytes(uint16_t port, size_t num_written,
+ size_t num_read)
+{
+ if (!start_of_exit_stats_interval)
+ return; /* Not initialized. */
+ exit_bytes_written[port] += num_written;
+ exit_bytes_read[port] += num_read;
+ log_debug(LD_HIST, "Written %lu bytes and read %lu bytes to/from an "
+ "exit connection to port %d.",
+ (unsigned long)num_written, (unsigned long)num_read, port);
+}
+
+/** Note that we opened an exit stream to <b>port</b>. */
+void
+rep_hist_note_exit_stream_opened(uint16_t port)
+{
+ if (!start_of_exit_stats_interval)
+ return; /* Not initialized. */
+ exit_streams[port]++;
+ log_debug(LD_HIST, "Opened exit stream to port %d", port);
+}
+
+/*** cell statistics ***/
+
+/** Start of the current buffer stats interval or 0 if we're not
+ * collecting buffer statistics. */
+static time_t start_of_buffer_stats_interval;
+
+/** Initialize buffer stats. */
+void
+rep_hist_buffer_stats_init(time_t now)
+{
+ start_of_buffer_stats_interval = now;
+}
+
+/** Statistics from a single circuit. Collected when the circuit closes, or
+ * when we flush statistics to disk. */
+typedef struct circ_buffer_stats_t {
+ /** Average number of cells in the circuit's queue */
+ double mean_num_cells_in_queue;
+ /** Average time a cell waits in the queue. */
+ double mean_time_cells_in_queue;
+ /** Total number of cells sent over this circuit */
+ uint32_t processed_cells;
+} circ_buffer_stats_t;
+
+/** List of circ_buffer_stats_t. */
+static smartlist_t *circuits_for_buffer_stats = NULL;
+
+/** Remember cell statistics <b>mean_num_cells_in_queue</b>,
+ * <b>mean_time_cells_in_queue</b>, and <b>processed_cells</b> of a
+ * circuit. */
+void
+rep_hist_add_buffer_stats(double mean_num_cells_in_queue,
+ double mean_time_cells_in_queue, uint32_t processed_cells)
+{
+ circ_buffer_stats_t *stats;
+ if (!start_of_buffer_stats_interval)
+ return; /* Not initialized. */
+ stats = tor_malloc_zero(sizeof(circ_buffer_stats_t));
+ stats->mean_num_cells_in_queue = mean_num_cells_in_queue;
+ stats->mean_time_cells_in_queue = mean_time_cells_in_queue;
+ stats->processed_cells = processed_cells;
+ if (!circuits_for_buffer_stats)
+ circuits_for_buffer_stats = smartlist_new();
+ smartlist_add(circuits_for_buffer_stats, stats);
+}
+
+/** Remember cell statistics for circuit <b>circ</b> at time
+ * <b>end_of_interval</b> and reset cell counters in case the circuit
+ * remains open in the next measurement interval. */
+void
+rep_hist_buffer_stats_add_circ(circuit_t *circ, time_t end_of_interval)
+{
+ time_t start_of_interval;
+ int interval_length;
+ or_circuit_t *orcirc;
+ double mean_num_cells_in_queue, mean_time_cells_in_queue;
+ uint32_t processed_cells;
+ if (CIRCUIT_IS_ORIGIN(circ))
+ return;
+ orcirc = TO_OR_CIRCUIT(circ);
+ if (!orcirc->processed_cells)
+ return;
+ start_of_interval = (circ->timestamp_created.tv_sec >
+ start_of_buffer_stats_interval) ?
+ (time_t)circ->timestamp_created.tv_sec :
+ start_of_buffer_stats_interval;
+ interval_length = (int) (end_of_interval - start_of_interval);
+ if (interval_length <= 0)
+ return;
+ processed_cells = orcirc->processed_cells;
+ /* 1000.0 for s -> ms; 2.0 because of app-ward and exit-ward queues */
+ mean_num_cells_in_queue = (double) orcirc->total_cell_waiting_time /
+ (double) interval_length / 1000.0 / 2.0;
+ mean_time_cells_in_queue =
+ (double) orcirc->total_cell_waiting_time /
+ (double) orcirc->processed_cells;
+ orcirc->total_cell_waiting_time = 0;
+ orcirc->processed_cells = 0;
+ rep_hist_add_buffer_stats(mean_num_cells_in_queue,
+ mean_time_cells_in_queue,
+ processed_cells);
+}
+
+/** Sorting helper: return -1, 1, or 0 based on comparison of two
+ * circ_buffer_stats_t */
+static int
+buffer_stats_compare_entries_(const void **_a, const void **_b)
+{
+ const circ_buffer_stats_t *a = *_a, *b = *_b;
+ if (a->processed_cells < b->processed_cells)
+ return 1;
+ else if (a->processed_cells > b->processed_cells)
+ return -1;
+ else
+ return 0;
+}
+
+/** Stop collecting cell stats in a way that we can re-start doing so in
+ * rep_hist_buffer_stats_init(). */
+void
+rep_hist_buffer_stats_term(void)
+{
+ rep_hist_reset_buffer_stats(0);
+}
+
+/** Clear history of circuit statistics and set the measurement interval
+ * start to <b>now</b>. */
+void
+rep_hist_reset_buffer_stats(time_t now)
+{
+ if (!circuits_for_buffer_stats)
+ circuits_for_buffer_stats = smartlist_new();
+ SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *,
+ stats, tor_free(stats));
+ smartlist_clear(circuits_for_buffer_stats);
+ start_of_buffer_stats_interval = now;
+}
+
+/** Return a newly allocated string containing the buffer statistics until
+ * <b>now</b>, or NULL if we're not collecting buffer stats. Caller must
+ * ensure start_of_buffer_stats_interval is in the past. */
+char *
+rep_hist_format_buffer_stats(time_t now)
+{
+#define SHARES 10
+ uint64_t processed_cells[SHARES];
+ uint32_t circs_in_share[SHARES];
+ int number_of_circuits, i;
+ double queued_cells[SHARES], time_in_queue[SHARES];
+ smartlist_t *processed_cells_strings, *queued_cells_strings,
+ *time_in_queue_strings;
+ char *processed_cells_string, *queued_cells_string,
+ *time_in_queue_string;
+ char t[ISO_TIME_LEN+1];
+ char *result;
+
+ if (!start_of_buffer_stats_interval)
+ return NULL; /* Not initialized. */
+
+ tor_assert(now >= start_of_buffer_stats_interval);
+
+ /* Calculate deciles if we saw at least one circuit. */
+ memset(processed_cells, 0, SHARES * sizeof(uint64_t));
+ memset(circs_in_share, 0, SHARES * sizeof(uint32_t));
+ memset(queued_cells, 0, SHARES * sizeof(double));
+ memset(time_in_queue, 0, SHARES * sizeof(double));
+ if (!circuits_for_buffer_stats)
+ circuits_for_buffer_stats = smartlist_new();
+ number_of_circuits = smartlist_len(circuits_for_buffer_stats);
+ if (number_of_circuits > 0) {
+ smartlist_sort(circuits_for_buffer_stats,
+ buffer_stats_compare_entries_);
+ i = 0;
+ SMARTLIST_FOREACH_BEGIN(circuits_for_buffer_stats,
+ circ_buffer_stats_t *, stats)
+ {
+ int share = i++ * SHARES / number_of_circuits;
+ processed_cells[share] += stats->processed_cells;
+ queued_cells[share] += stats->mean_num_cells_in_queue;
+ time_in_queue[share] += stats->mean_time_cells_in_queue;
+ circs_in_share[share]++;
+ }
+ SMARTLIST_FOREACH_END(stats);
+ }
+
+ /* Write deciles to strings. */
+ processed_cells_strings = smartlist_new();
+ queued_cells_strings = smartlist_new();
+ time_in_queue_strings = smartlist_new();
+ for (i = 0; i < SHARES; i++) {
+ smartlist_add_asprintf(processed_cells_strings,
+ "%"PRIu64, !circs_in_share[i] ? 0 :
+ (processed_cells[i] /
+ circs_in_share[i]));
+ }
+ for (i = 0; i < SHARES; i++) {
+ smartlist_add_asprintf(queued_cells_strings, "%.2f",
+ circs_in_share[i] == 0 ? 0.0 :
+ queued_cells[i] / (double) circs_in_share[i]);
+ }
+ for (i = 0; i < SHARES; i++) {
+ smartlist_add_asprintf(time_in_queue_strings, "%.0f",
+ circs_in_share[i] == 0 ? 0.0 :
+ time_in_queue[i] / (double) circs_in_share[i]);
+ }
+
+ /* Join all observations in single strings. */
+ processed_cells_string = smartlist_join_strings(processed_cells_strings,
+ ",", 0, NULL);
+ queued_cells_string = smartlist_join_strings(queued_cells_strings,
+ ",", 0, NULL);
+ time_in_queue_string = smartlist_join_strings(time_in_queue_strings,
+ ",", 0, NULL);
+ SMARTLIST_FOREACH(processed_cells_strings, char *, cp, tor_free(cp));
+ SMARTLIST_FOREACH(queued_cells_strings, char *, cp, tor_free(cp));
+ SMARTLIST_FOREACH(time_in_queue_strings, char *, cp, tor_free(cp));
+ smartlist_free(processed_cells_strings);
+ smartlist_free(queued_cells_strings);
+ smartlist_free(time_in_queue_strings);
+
+ /* Put everything together. */
+ format_iso_time(t, now);
+ tor_asprintf(&result, "cell-stats-end %s (%d s)\n"
+ "cell-processed-cells %s\n"
+ "cell-queued-cells %s\n"
+ "cell-time-in-queue %s\n"
+ "cell-circuits-per-decile %d\n",
+ t, (unsigned) (now - start_of_buffer_stats_interval),
+ processed_cells_string,
+ queued_cells_string,
+ time_in_queue_string,
+ CEIL_DIV(number_of_circuits, SHARES));
+ tor_free(processed_cells_string);
+ tor_free(queued_cells_string);
+ tor_free(time_in_queue_string);
+ return result;
+#undef SHARES
+}
+
+/** If 24 hours have passed since the beginning of the current buffer
+ * stats period, write buffer stats to $DATADIR/stats/buffer-stats
+ * (possibly overwriting an existing file) and reset counters. Return
+ * when we would next want to write buffer stats or 0 if we never want to
+ * write. */
+time_t
+rep_hist_buffer_stats_write(time_t now)
+{
+ char *str = NULL;
+
+ if (!start_of_buffer_stats_interval)
+ return 0; /* Not initialized. */
+ if (start_of_buffer_stats_interval + WRITE_STATS_INTERVAL > now)
+ goto done; /* Not ready to write */
+
+ /* Add open circuits to the history. */
+ SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+ rep_hist_buffer_stats_add_circ(circ, now);
+ }
+ SMARTLIST_FOREACH_END(circ);
+
+ /* Generate history string. */
+ str = rep_hist_format_buffer_stats(now);
+
+ /* Reset both buffer history and counters of open circuits. */
+ rep_hist_reset_buffer_stats(now);
+
+ /* Try to write to disk. */
+ if (!check_or_create_data_subdir("stats")) {
+ write_to_data_subdir("stats", "buffer-stats", str, "buffer statistics");
+ }
+
+ done:
+ tor_free(str);
+ return start_of_buffer_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/*** Descriptor serving statistics ***/
+
+/** Digestmap to track which descriptors were downloaded this stats
+ * collection interval. It maps descriptor digest to pointers to 1,
+ * effectively turning this into a list. */
+static digestmap_t *served_descs = NULL;
+
+/** Number of how many descriptors were downloaded in total during this
+ * interval. */
+static unsigned long total_descriptor_downloads;
+
+/** Start time of served descs stats or 0 if we're not collecting those. */
+static time_t start_of_served_descs_stats_interval;
+
+/** Initialize descriptor stats. */
+void
+rep_hist_desc_stats_init(time_t now)
+{
+ if (served_descs) {
+ log_warn(LD_BUG, "Called rep_hist_desc_stats_init() when desc stats were "
+ "already initialized. This is probably harmless.");
+ return; // Already initialized
+ }
+ served_descs = digestmap_new();
+ total_descriptor_downloads = 0;
+ start_of_served_descs_stats_interval = now;
+}
+
+/** Reset served descs stats to empty, starting a new interval <b>now</b>. */
+static void
+rep_hist_reset_desc_stats(time_t now)
+{
+ rep_hist_desc_stats_term();
+ rep_hist_desc_stats_init(now);
+}
+
+/** Stop collecting served descs stats, so that rep_hist_desc_stats_init() is
+ * safe to be called again. */
+void
+rep_hist_desc_stats_term(void)
+{
+ digestmap_free(served_descs, NULL);
+ served_descs = NULL;
+ start_of_served_descs_stats_interval = 0;
+ total_descriptor_downloads = 0;
+}
+
+/** Helper for rep_hist_desc_stats_write(). Return a newly allocated string
+ * containing the served desc statistics until now, or NULL if we're not
+ * collecting served desc stats. Caller must ensure that now is not before
+ * start_of_served_descs_stats_interval. */
+static char *
+rep_hist_format_desc_stats(time_t now)
+{
+ char t[ISO_TIME_LEN+1];
+ char *result;
+
+ digestmap_iter_t *iter;
+ const char *key;
+ void *val;
+ unsigned size;
+ int *vals, max = 0, q3 = 0, md = 0, q1 = 0, min = 0;
+ int n = 0;
+
+ if (!start_of_served_descs_stats_interval)
+ return NULL;
+
+ size = digestmap_size(served_descs);
+ if (size > 0) {
+ vals = tor_calloc(size, sizeof(int));
+ for (iter = digestmap_iter_init(served_descs);
+ !digestmap_iter_done(iter);
+ iter = digestmap_iter_next(served_descs, iter)) {
+ uintptr_t count;
+ digestmap_iter_get(iter, &key, &val);
+ count = (uintptr_t)val;
+ vals[n++] = (int)count;
+ (void)key;
+ }
+ max = find_nth_int(vals, size, size-1);
+ q3 = find_nth_int(vals, size, (3*size-1)/4);
+ md = find_nth_int(vals, size, (size-1)/2);
+ q1 = find_nth_int(vals, size, (size-1)/4);
+ min = find_nth_int(vals, size, 0);
+ tor_free(vals);
+ }
+
+ format_iso_time(t, now);
+
+ tor_asprintf(&result,
+ "served-descs-stats-end %s (%d s) total=%lu unique=%u "
+ "max=%d q3=%d md=%d q1=%d min=%d\n",
+ t,
+ (unsigned) (now - start_of_served_descs_stats_interval),
+ total_descriptor_downloads,
+ size, max, q3, md, q1, min);
+
+ return result;
+}
+
+/** If WRITE_STATS_INTERVAL seconds have passed since the beginning of
+ * the current served desc stats interval, write the stats to
+ * $DATADIR/stats/served-desc-stats (possibly appending to an existing file)
+ * and reset the state for the next interval. Return when we would next want
+ * to write served desc stats or 0 if we won't want to write. */
+time_t
+rep_hist_desc_stats_write(time_t now)
+{
+ char *filename = NULL, *str = NULL;
+
+ if (!start_of_served_descs_stats_interval)
+ return 0; /* We're not collecting stats. */
+ if (start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL > now)
+ return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL;
+
+ str = rep_hist_format_desc_stats(now);
+ tor_assert(str != NULL);
+
+ if (check_or_create_data_subdir("stats") < 0) {
+ goto done;
+ }
+ filename = get_datadir_fname2("stats", "served-desc-stats");
+ if (append_bytes_to_file(filename, str, strlen(str), 0) < 0)
+ log_warn(LD_HIST, "Unable to write served descs statistics to disk!");
+
+ rep_hist_reset_desc_stats(now);
+
+ done:
+ tor_free(filename);
+ tor_free(str);
+ return start_of_served_descs_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/** Called to note that we've served a given descriptor (by
+ * digest). Increments the count of descriptors served, and the number
+ * of times we've served this descriptor. */
+void
+rep_hist_note_desc_served(const char * desc)
+{
+ void *val;
+ uintptr_t count;
+ if (!served_descs)
+ return; // We're not collecting stats
+ val = digestmap_get(served_descs, desc);
+ count = (uintptr_t)val;
+ if (count != INT_MAX)
+ ++count;
+ digestmap_set(served_descs, desc, (void*)count);
+ total_descriptor_downloads++;
+}
+
+/*** Connection statistics ***/
+
+/** Start of the current connection stats interval or 0 if we're not
+ * collecting connection statistics. */
+static time_t start_of_conn_stats_interval;
+
+/** Initialize connection stats. */
+void
+rep_hist_conn_stats_init(time_t now)
+{
+ start_of_conn_stats_interval = now;
+}
+
+/* Count connections that we read and wrote less than these many bytes
+ * from/to as below threshold. */
+#define BIDI_THRESHOLD 20480
+
+/* Count connections that we read or wrote at least this factor as many
+ * bytes from/to than we wrote or read to/from as mostly reading or
+ * writing. */
+#define BIDI_FACTOR 10
+
+/* Interval length in seconds for considering read and written bytes for
+ * connection stats. */
+#define BIDI_INTERVAL 10
+
+/** Start of next BIDI_INTERVAL second interval. */
+static time_t bidi_next_interval = 0;
+
+/** Number of connections that we read and wrote less than BIDI_THRESHOLD
+ * bytes from/to in BIDI_INTERVAL seconds. */
+static uint32_t below_threshold = 0;
+
+/** Number of connections that we read at least BIDI_FACTOR times more
+ * bytes from than we wrote to in BIDI_INTERVAL seconds. */
+static uint32_t mostly_read = 0;
+
+/** Number of connections that we wrote at least BIDI_FACTOR times more
+ * bytes to than we read from in BIDI_INTERVAL seconds. */
+static uint32_t mostly_written = 0;
+
+/** Number of connections that we read and wrote at least BIDI_THRESHOLD
+ * bytes from/to, but not BIDI_FACTOR times more in either direction in
+ * BIDI_INTERVAL seconds. */
+static uint32_t both_read_and_written = 0;
+
+/** Entry in a map from connection ID to the number of read and written
+ * bytes on this connection in a BIDI_INTERVAL second interval. */
+typedef struct bidi_map_entry_t {
+ HT_ENTRY(bidi_map_entry_t) node;
+ uint64_t conn_id; /**< Connection ID */
+ size_t read; /**< Number of read bytes */
+ size_t written; /**< Number of written bytes */
+} bidi_map_entry_t;
+
+/** Map of OR connections together with the number of read and written
+ * bytes in the current BIDI_INTERVAL second interval. */
+static HT_HEAD(bidimap, bidi_map_entry_t) bidi_map =
+ HT_INITIALIZER();
+
+static int
+bidi_map_ent_eq(const bidi_map_entry_t *a, const bidi_map_entry_t *b)
+{
+ return a->conn_id == b->conn_id;
+}
+
+/* DOCDOC bidi_map_ent_hash */
+static unsigned
+bidi_map_ent_hash(const bidi_map_entry_t *entry)
+{
+ return (unsigned) entry->conn_id;
+}
+
+HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
+ bidi_map_ent_eq)
+HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
+ bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_)
+
+/* DOCDOC bidi_map_free */
+static void
+bidi_map_free_all(void)
+{
+ bidi_map_entry_t **ptr, **next, *ent;
+ for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) {
+ ent = *ptr;
+ next = HT_NEXT_RMV(bidimap, &bidi_map, ptr);
+ tor_free(ent);
+ }
+ HT_CLEAR(bidimap, &bidi_map);
+}
+
+/** Reset counters for conn statistics. */
+void
+rep_hist_reset_conn_stats(time_t now)
+{
+ start_of_conn_stats_interval = now;
+ below_threshold = 0;
+ mostly_read = 0;
+ mostly_written = 0;
+ both_read_and_written = 0;
+ bidi_map_free_all();
+}
+
+/** Stop collecting connection stats in a way that we can re-start doing
+ * so in rep_hist_conn_stats_init(). */
+void
+rep_hist_conn_stats_term(void)
+{
+ rep_hist_reset_conn_stats(0);
+}
+
+/** We read <b>num_read</b> bytes and wrote <b>num_written</b> from/to OR
+ * connection <b>conn_id</b> in second <b>when</b>. If this is the first
+ * observation in a new interval, sum up the last observations. Add bytes
+ * for this connection. */
+void
+rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read,
+ size_t num_written, time_t when)
+{
+ if (!start_of_conn_stats_interval)
+ return;
+ /* Initialize */
+ if (bidi_next_interval == 0)
+ bidi_next_interval = when + BIDI_INTERVAL;
+ /* Sum up last period's statistics */
+ if (when >= bidi_next_interval) {
+ bidi_map_entry_t **ptr, **next, *ent;
+ for (ptr = HT_START(bidimap, &bidi_map); ptr; ptr = next) {
+ ent = *ptr;
+ if (ent->read + ent->written < BIDI_THRESHOLD)
+ below_threshold++;
+ else if (ent->read >= ent->written * BIDI_FACTOR)
+ mostly_read++;
+ else if (ent->written >= ent->read * BIDI_FACTOR)
+ mostly_written++;
+ else
+ both_read_and_written++;
+ next = HT_NEXT_RMV(bidimap, &bidi_map, ptr);
+ tor_free(ent);
+ }
+ while (when >= bidi_next_interval)
+ bidi_next_interval += BIDI_INTERVAL;
+ log_info(LD_GENERAL, "%d below threshold, %d mostly read, "
+ "%d mostly written, %d both read and written.",
+ below_threshold, mostly_read, mostly_written,
+ both_read_and_written);
+ }
+ /* Add this connection's bytes. */
+ if (num_read > 0 || num_written > 0) {
+ bidi_map_entry_t *entry, lookup;
+ lookup.conn_id = conn_id;
+ entry = HT_FIND(bidimap, &bidi_map, &lookup);
+ if (entry) {
+ entry->written += num_written;
+ entry->read += num_read;
+ } else {
+ entry = tor_malloc_zero(sizeof(bidi_map_entry_t));
+ entry->conn_id = conn_id;
+ entry->written = num_written;
+ entry->read = num_read;
+ HT_INSERT(bidimap, &bidi_map, entry);
+ }
+ }
+}
+
+/** Return a newly allocated string containing the connection statistics
+ * until <b>now</b>, or NULL if we're not collecting conn stats. Caller must
+ * ensure start_of_conn_stats_interval is in the past. */
+char *
+rep_hist_format_conn_stats(time_t now)
+{
+ char *result, written[ISO_TIME_LEN+1];
+
+ if (!start_of_conn_stats_interval)
+ return NULL; /* Not initialized. */
+
+ tor_assert(now >= start_of_conn_stats_interval);
+
+ format_iso_time(written, now);
+ tor_asprintf(&result, "conn-bi-direct %s (%d s) %d,%d,%d,%d\n",
+ written,
+ (unsigned) (now - start_of_conn_stats_interval),
+ below_threshold,
+ mostly_read,
+ mostly_written,
+ both_read_and_written);
+ return result;
+}
+
+/** If 24 hours have passed since the beginning of the current conn stats
+ * period, write conn stats to $DATADIR/stats/conn-stats (possibly
+ * overwriting an existing file) and reset counters. Return when we would
+ * next want to write conn stats or 0 if we never want to write. */
+time_t
+rep_hist_conn_stats_write(time_t now)
+{
+ char *str = NULL;
+
+ if (!start_of_conn_stats_interval)
+ return 0; /* Not initialized. */
+ if (start_of_conn_stats_interval + WRITE_STATS_INTERVAL > now)
+ goto done; /* Not ready to write */
+
+ /* Generate history string. */
+ str = rep_hist_format_conn_stats(now);
+
+ /* Reset counters. */
+ rep_hist_reset_conn_stats(now);
+
+ /* Try to write to disk. */
+ if (!check_or_create_data_subdir("stats")) {
+ write_to_data_subdir("stats", "conn-stats", str, "connection statistics");
+ }
+
+ done:
+ tor_free(str);
+ return start_of_conn_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+/** Internal statistics to track how many requests of each type of
+ * handshake we've received, and how many we've assigned to cpuworkers.
+ * Useful for seeing trends in cpu load.
+ * @{ */
+STATIC int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1] = {0};
+STATIC int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1] = {0};
+/**@}*/
+
+/** A new onionskin (using the <b>type</b> handshake) has arrived. */
+void
+rep_hist_note_circuit_handshake_requested(uint16_t type)
+{
+ if (type <= MAX_ONION_HANDSHAKE_TYPE)
+ onion_handshakes_requested[type]++;
+}
+
+/** We've sent an onionskin (using the <b>type</b> handshake) to a
+ * cpuworker. */
+void
+rep_hist_note_circuit_handshake_assigned(uint16_t type)
+{
+ if (type <= MAX_ONION_HANDSHAKE_TYPE)
+ onion_handshakes_assigned[type]++;
+}
+
+/** Log our onionskin statistics since the last time we were called. */
+void
+rep_hist_log_circuit_handshake_stats(time_t now)
+{
+ (void)now;
+ log_notice(LD_HEARTBEAT, "Circuit handshake stats since last time: "
+ "%d/%d TAP, %d/%d NTor.",
+ onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_TAP],
+ onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP],
+ onion_handshakes_assigned[ONION_HANDSHAKE_TYPE_NTOR],
+ onion_handshakes_requested[ONION_HANDSHAKE_TYPE_NTOR]);
+ memset(onion_handshakes_assigned, 0, sizeof(onion_handshakes_assigned));
+ memset(onion_handshakes_requested, 0, sizeof(onion_handshakes_requested));
+}
+
+/* Hidden service statistics section */
+
+/** Start of the current hidden service stats interval or 0 if we're
+ * not collecting hidden service statistics. */
+static time_t start_of_hs_stats_interval;
+
+/** Carries the various hidden service statistics, and any other
+ * information needed. */
+typedef struct hs_stats_t {
+ /** How many relay cells have we seen as rendezvous points? */
+ uint64_t rp_relay_cells_seen;
+
+ /** Set of unique public key digests we've seen this stat period
+ * (could also be implemented as sorted smartlist). */
+ digestmap_t *onions_seen_this_period;
+} hs_stats_t;
+
+/** Our statistics structure singleton. */
+static hs_stats_t *hs_stats = NULL;
+
+/** Allocate, initialize and return an hs_stats_t structure. */
+static hs_stats_t *
+hs_stats_new(void)
+{
+ hs_stats_t *new_hs_stats = tor_malloc_zero(sizeof(hs_stats_t));
+ new_hs_stats->onions_seen_this_period = digestmap_new();
+
+ return new_hs_stats;
+}
+
+#define hs_stats_free(val) \
+ FREE_AND_NULL(hs_stats_t, hs_stats_free_, (val))
+
+/** Free an hs_stats_t structure. */
+static void
+hs_stats_free_(hs_stats_t *victim_hs_stats)
+{
+ if (!victim_hs_stats) {
+ return;
+ }
+
+ digestmap_free(victim_hs_stats->onions_seen_this_period, NULL);
+ tor_free(victim_hs_stats);
+}
+
+/** Initialize hidden service statistics. */
+void
+rep_hist_hs_stats_init(time_t now)
+{
+ if (!hs_stats) {
+ hs_stats = hs_stats_new();
+ }
+
+ start_of_hs_stats_interval = now;
+}
+
+/** Clear history of hidden service statistics and set the measurement
+ * interval start to <b>now</b>. */
+static void
+rep_hist_reset_hs_stats(time_t now)
+{
+ if (!hs_stats) {
+ hs_stats = hs_stats_new();
+ }
+
+ hs_stats->rp_relay_cells_seen = 0;
+
+ digestmap_free(hs_stats->onions_seen_this_period, NULL);
+ hs_stats->onions_seen_this_period = digestmap_new();
+
+ start_of_hs_stats_interval = now;
+}
+
+/** Stop collecting hidden service stats in a way that we can re-start
+ * doing so in rep_hist_buffer_stats_init(). */
+void
+rep_hist_hs_stats_term(void)
+{
+ rep_hist_reset_hs_stats(0);
+}
+
+/** We saw a new HS relay cell, Count it! */
+void
+rep_hist_seen_new_rp_cell(void)
+{
+ if (!hs_stats) {
+ return; // We're not collecting stats
+ }
+
+ hs_stats->rp_relay_cells_seen++;
+}
+
+/** As HSDirs, we saw another hidden service with public key
+ * <b>pubkey</b>. Check whether we have counted it before, if not
+ * count it now! */
+void
+rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey)
+{
+ char pubkey_hash[DIGEST_LEN];
+
+ if (!hs_stats) {
+ return; // We're not collecting stats
+ }
+
+ /* Get the digest of the pubkey which will be used to detect whether
+ we've seen this hidden service before or not. */
+ if (crypto_pk_get_digest(pubkey, pubkey_hash) < 0) {
+ /* This fail should not happen; key has been validated by
+ descriptor parsing code first. */
+ return;
+ }
+
+ /* Check if this is the first time we've seen this hidden
+ service. If it is, count it as new. */
+ if (!digestmap_get(hs_stats->onions_seen_this_period,
+ pubkey_hash)) {
+ digestmap_set(hs_stats->onions_seen_this_period,
+ pubkey_hash, (void*)(uintptr_t)1);
+ }
+}
+
+/* The number of cells that are supposed to be hidden from the adversary
+ * by adding noise from the Laplace distribution. This value, divided by
+ * EPSILON, is Laplace parameter b. It must be greather than 0. */
+#define REND_CELLS_DELTA_F 2048
+/* Security parameter for obfuscating number of cells with a value between
+ * ]0.0, 1.0]. Smaller values obfuscate observations more, but at the same
+ * time make statistics less usable. */
+#define REND_CELLS_EPSILON 0.3
+/* The number of cells that are supposed to be hidden from the adversary
+ * by rounding up to the next multiple of this number. */
+#define REND_CELLS_BIN_SIZE 1024
+/* The number of service identities that are supposed to be hidden from the
+ * adversary by adding noise from the Laplace distribution. This value,
+ * divided by EPSILON, is Laplace parameter b. It must be greater than 0. */
+#define ONIONS_SEEN_DELTA_F 8
+/* Security parameter for obfuscating number of service identities with a
+ * value between ]0.0, 1.0]. Smaller values obfuscate observations more, but
+ * at the same time make statistics less usable. */
+#define ONIONS_SEEN_EPSILON 0.3
+/* The number of service identities that are supposed to be hidden from
+ * the adversary by rounding up to the next multiple of this number. */
+#define ONIONS_SEEN_BIN_SIZE 8
+
+/** Allocate and return a string containing hidden service stats that
+ * are meant to be placed in the extra-info descriptor. */
+static char *
+rep_hist_format_hs_stats(time_t now)
+{
+ char t[ISO_TIME_LEN+1];
+ char *hs_stats_string;
+ int64_t obfuscated_cells_seen;
+ int64_t obfuscated_onions_seen;
+
+ uint64_t rounded_cells_seen
+ = round_uint64_to_next_multiple_of(hs_stats->rp_relay_cells_seen,
+ REND_CELLS_BIN_SIZE);
+ rounded_cells_seen = MIN(rounded_cells_seen, INT64_MAX);
+ obfuscated_cells_seen = add_laplace_noise((int64_t)rounded_cells_seen,
+ crypto_rand_double(),
+ REND_CELLS_DELTA_F, REND_CELLS_EPSILON);
+
+ uint64_t rounded_onions_seen =
+ round_uint64_to_next_multiple_of((size_t)digestmap_size(
+ hs_stats->onions_seen_this_period),
+ ONIONS_SEEN_BIN_SIZE);
+ rounded_onions_seen = MIN(rounded_onions_seen, INT64_MAX);
+ obfuscated_onions_seen = add_laplace_noise((int64_t)rounded_onions_seen,
+ crypto_rand_double(), ONIONS_SEEN_DELTA_F,
+ ONIONS_SEEN_EPSILON);
+
+ format_iso_time(t, now);
+ tor_asprintf(&hs_stats_string, "hidserv-stats-end %s (%d s)\n"
+ "hidserv-rend-relayed-cells %"PRId64" delta_f=%d "
+ "epsilon=%.2f bin_size=%d\n"
+ "hidserv-dir-onions-seen %"PRId64" delta_f=%d "
+ "epsilon=%.2f bin_size=%d\n",
+ t, (unsigned) (now - start_of_hs_stats_interval),
+ (obfuscated_cells_seen), REND_CELLS_DELTA_F,
+ REND_CELLS_EPSILON, REND_CELLS_BIN_SIZE,
+ (obfuscated_onions_seen),
+ ONIONS_SEEN_DELTA_F,
+ ONIONS_SEEN_EPSILON, ONIONS_SEEN_BIN_SIZE);
+
+ return hs_stats_string;
+}
+
+/** If 24 hours have passed since the beginning of the current HS
+ * stats period, write buffer stats to $DATADIR/stats/hidserv-stats
+ * (possibly overwriting an existing file) and reset counters. Return
+ * when we would next want to write buffer stats or 0 if we never want to
+ * write. */
+time_t
+rep_hist_hs_stats_write(time_t now)
+{
+ char *str = NULL;
+
+ if (!start_of_hs_stats_interval) {
+ return 0; /* Not initialized. */
+ }
+
+ if (start_of_hs_stats_interval + WRITE_STATS_INTERVAL > now) {
+ goto done; /* Not ready to write */
+ }
+
+ /* Generate history string. */
+ str = rep_hist_format_hs_stats(now);
+
+ /* Reset HS history. */
+ rep_hist_reset_hs_stats(now);
+
+ /* Try to write to disk. */
+ if (!check_or_create_data_subdir("stats")) {
+ write_to_data_subdir("stats", "hidserv-stats", str,
+ "hidden service stats");
+ }
+
+ done:
+ tor_free(str);
+ return start_of_hs_stats_interval + WRITE_STATS_INTERVAL;
+}
+
+static uint64_t link_proto_count[MAX_LINK_PROTO+1][2];
+
+/** Note that we negotiated link protocol version <b>link_proto</b>, on
+ * a connection that started here iff <b>started_here</b> is true.
+ */
+void
+rep_hist_note_negotiated_link_proto(unsigned link_proto, int started_here)
+{
+ started_here = !!started_here; /* force to 0 or 1 */
+ if (link_proto > MAX_LINK_PROTO) {
+ log_warn(LD_BUG, "Can't log link protocol %u", link_proto);
+ return;
+ }
+
+ link_proto_count[link_proto][started_here]++;
+}
+
+/**
+ * Update the maximum count of total pending channel padding timers
+ * in this period.
+ */
+void
+rep_hist_padding_count_timers(uint64_t num_timers)
+{
+ if (num_timers > padding_current.maximum_chanpad_timers) {
+ padding_current.maximum_chanpad_timers = num_timers;
+ }
+}
+
+/**
+ * Count a cell that we sent for padding overhead statistics.
+ *
+ * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
+ * counted for PADDING_TYPE_TOTAL.
+ */
+void
+rep_hist_padding_count_write(padding_type_t type)
+{
+ switch (type) {
+ case PADDING_TYPE_DROP:
+ padding_current.write_drop_cell_count++;
+ break;
+ case PADDING_TYPE_CELL:
+ padding_current.write_pad_cell_count++;
+ break;
+ case PADDING_TYPE_TOTAL:
+ padding_current.write_cell_count++;
+ break;
+ case PADDING_TYPE_ENABLED_TOTAL:
+ padding_current.enabled_write_cell_count++;
+ break;
+ case PADDING_TYPE_ENABLED_CELL:
+ padding_current.enabled_write_pad_cell_count++;
+ break;
+ }
+}
+
+/**
+ * Count a cell that we've received for padding overhead statistics.
+ *
+ * RELAY_COMMAND_DROP and CELL_PADDING are accounted separately. Both should be
+ * counted for PADDING_TYPE_TOTAL.
+ */
+void
+rep_hist_padding_count_read(padding_type_t type)
+{
+ switch (type) {
+ case PADDING_TYPE_DROP:
+ padding_current.read_drop_cell_count++;
+ break;
+ case PADDING_TYPE_CELL:
+ padding_current.read_pad_cell_count++;
+ break;
+ case PADDING_TYPE_TOTAL:
+ padding_current.read_cell_count++;
+ break;
+ case PADDING_TYPE_ENABLED_TOTAL:
+ padding_current.enabled_read_cell_count++;
+ break;
+ case PADDING_TYPE_ENABLED_CELL:
+ padding_current.enabled_read_pad_cell_count++;
+ break;
+ }
+}
+
+/**
+ * Reset our current padding statistics. Called once every 24 hours.
+ */
+void
+rep_hist_reset_padding_counts(void)
+{
+ memset(&padding_current, 0, sizeof(padding_current));
+}
+
+/**
+ * Copy our current cell counts into a structure for listing in our
+ * extra-info descriptor. Also perform appropriate rounding and redaction.
+ *
+ * This function is called once every 24 hours.
+ */
+#define MIN_CELL_COUNTS_TO_PUBLISH 1
+#define ROUND_CELL_COUNTS_TO 10000
+void
+rep_hist_prep_published_padding_counts(time_t now)
+{
+ memcpy(&padding_published, &padding_current, sizeof(padding_published));
+
+ if (padding_published.read_cell_count < MIN_CELL_COUNTS_TO_PUBLISH ||
+ padding_published.write_cell_count < MIN_CELL_COUNTS_TO_PUBLISH) {
+ memset(&padding_published, 0, sizeof(padding_published));
+ return;
+ }
+
+ format_iso_time(padding_published.first_published_at, now);
+#define ROUND_AND_SET_COUNT(x) (x) = round_uint64_to_next_multiple_of((x), \
+ ROUND_CELL_COUNTS_TO)
+ ROUND_AND_SET_COUNT(padding_published.read_pad_cell_count);
+ ROUND_AND_SET_COUNT(padding_published.write_pad_cell_count);
+ ROUND_AND_SET_COUNT(padding_published.read_drop_cell_count);
+ ROUND_AND_SET_COUNT(padding_published.write_drop_cell_count);
+ ROUND_AND_SET_COUNT(padding_published.write_cell_count);
+ ROUND_AND_SET_COUNT(padding_published.read_cell_count);
+ ROUND_AND_SET_COUNT(padding_published.enabled_read_cell_count);
+ ROUND_AND_SET_COUNT(padding_published.enabled_read_pad_cell_count);
+ ROUND_AND_SET_COUNT(padding_published.enabled_write_cell_count);
+ ROUND_AND_SET_COUNT(padding_published.enabled_write_pad_cell_count);
+#undef ROUND_AND_SET_COUNT
+}
+
+/**
+ * Returns an allocated string for extra-info documents for publishing
+ * padding statistics from the last 24 hour interval.
+ */
+char *
+rep_hist_get_padding_count_lines(void)
+{
+ char *result = NULL;
+
+ if (!padding_published.read_cell_count ||
+ !padding_published.write_cell_count) {
+ return NULL;
+ }
+
+ tor_asprintf(&result, "padding-counts %s (%d s)"
+ " bin-size=%"PRIu64
+ " write-drop=%"PRIu64
+ " write-pad=%"PRIu64
+ " write-total=%"PRIu64
+ " read-drop=%"PRIu64
+ " read-pad=%"PRIu64
+ " read-total=%"PRIu64
+ " enabled-read-pad=%"PRIu64
+ " enabled-read-total=%"PRIu64
+ " enabled-write-pad=%"PRIu64
+ " enabled-write-total=%"PRIu64
+ " max-chanpad-timers=%"PRIu64
+ "\n",
+ padding_published.first_published_at,
+ REPHIST_CELL_PADDING_COUNTS_INTERVAL,
+ (uint64_t)ROUND_CELL_COUNTS_TO,
+ (padding_published.write_drop_cell_count),
+ (padding_published.write_pad_cell_count),
+ (padding_published.write_cell_count),
+ (padding_published.read_drop_cell_count),
+ (padding_published.read_pad_cell_count),
+ (padding_published.read_cell_count),
+ (padding_published.enabled_read_pad_cell_count),
+ (padding_published.enabled_read_cell_count),
+ (padding_published.enabled_write_pad_cell_count),
+ (padding_published.enabled_write_cell_count),
+ (padding_published.maximum_chanpad_timers)
+ );
+
+ return result;
+}
+
+/** Log a heartbeat message explaining how many connections of each link
+ * protocol version we have used.
+ */
+void
+rep_hist_log_link_protocol_counts(void)
+{
+ log_notice(LD_HEARTBEAT,
+ "Since startup, we have initiated "
+ "%"PRIu64" v1 connections, "
+ "%"PRIu64" v2 connections, "
+ "%"PRIu64" v3 connections, and "
+ "%"PRIu64" v4 connections; and received "
+ "%"PRIu64" v1 connections, "
+ "%"PRIu64" v2 connections, "
+ "%"PRIu64" v3 connections, and "
+ "%"PRIu64" v4 connections.",
+ (link_proto_count[1][1]),
+ (link_proto_count[2][1]),
+ (link_proto_count[3][1]),
+ (link_proto_count[4][1]),
+ (link_proto_count[1][0]),
+ (link_proto_count[2][0]),
+ (link_proto_count[3][0]),
+ (link_proto_count[4][0]));
+}
+
+/** Free all storage held by the OR/link history caches, by the
+ * bandwidth history arrays, by the port history, or by statistics . */
+void
+rep_hist_free_all(void)
+{
+ hs_stats_free(hs_stats);
+ digestmap_free(history_map, free_or_history);
+
+ bw_array_free(read_array);
+ read_array = NULL;
+
+ bw_array_free(write_array);
+ write_array = NULL;
+
+ bw_array_free(dir_read_array);
+ dir_read_array = NULL;
+
+ bw_array_free(dir_write_array);
+ dir_write_array = NULL;
+
+ tor_free(exit_bytes_read);
+ tor_free(exit_bytes_written);
+ tor_free(exit_streams);
+ predicted_ports_free_all();
+ bidi_map_free_all();
+
+ if (circuits_for_buffer_stats) {
+ SMARTLIST_FOREACH(circuits_for_buffer_stats, circ_buffer_stats_t *, s,
+ tor_free(s));
+ smartlist_free(circuits_for_buffer_stats);
+ circuits_for_buffer_stats = NULL;
+ }
+ rep_hist_desc_stats_term();
+ total_descriptor_downloads = 0;
+
+ tor_assert_nonfatal(rephist_total_alloc == 0);
+ tor_assert_nonfatal_once(rephist_total_num == 0);
+}
diff --cc src/feature/stats/rephist.h
index 06a5e4821,000000000..67a015a4c
mode 100644,000000..100644
--- a/src/feature/stats/rephist.h
+++ b/src/feature/stats/rephist.h
@@@ -1,140 -1,0 +1,140 @@@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rephist.h
+ * \brief Header file for rephist.c.
+ **/
+
+#ifndef TOR_REPHIST_H
+#define TOR_REPHIST_H
+
+void rep_hist_init(void);
+void rep_hist_dump_stats(time_t now, int severity);
+void rep_hist_note_bytes_read(size_t num_bytes, time_t when);
+void rep_hist_note_bytes_written(size_t num_bytes, time_t when);
+
+void rep_hist_make_router_pessimal(const char *id, time_t when);
+
+void rep_hist_note_dir_bytes_read(size_t num_bytes, time_t when);
+void rep_hist_note_dir_bytes_written(size_t num_bytes, time_t when);
+
- int rep_hist_bandwidth_assess(void);
++MOCK_DECL(int, rep_hist_bandwidth_assess, (void));
+char *rep_hist_get_bandwidth_lines(void);
+void rep_hist_update_state(or_state_t *state);
+int rep_hist_load_state(or_state_t *state, char **err);
+void rep_history_clean(time_t before);
+
+void rep_hist_note_router_reachable(const char *id, const tor_addr_t *at_addr,
+ const uint16_t at_port, time_t when);
+void rep_hist_note_router_unreachable(const char *id, time_t when);
+int rep_hist_record_mtbf_data(time_t now, int missing_means_down);
+int rep_hist_load_mtbf_data(time_t now);
+
+time_t rep_hist_downrate_old_runs(time_t now);
+long rep_hist_get_uptime(const char *id, time_t when);
+double rep_hist_get_stability(const char *id, time_t when);
+double rep_hist_get_weighted_fractional_uptime(const char *id, time_t when);
+long rep_hist_get_weighted_time_known(const char *id, time_t when);
+int rep_hist_have_measured_enough_stability(void);
+
+void predicted_ports_init(void);
+void rep_hist_note_used_port(time_t now, uint16_t port);
+smartlist_t *rep_hist_get_predicted_ports(time_t now);
+void rep_hist_remove_predicted_ports(const smartlist_t *rmv_ports);
+void rep_hist_note_used_resolve(time_t now);
+void rep_hist_note_used_internal(time_t now, int need_uptime,
+ int need_capacity);
+int rep_hist_get_predicted_internal(time_t now, int *need_uptime,
+ int *need_capacity);
+
+int any_predicted_circuits(time_t now);
+int rep_hist_circbuilding_dormant(time_t now);
+int predicted_ports_prediction_time_remaining(time_t now);
+
+void rep_hist_exit_stats_init(time_t now);
+void rep_hist_reset_exit_stats(time_t now);
+void rep_hist_exit_stats_term(void);
+char *rep_hist_format_exit_stats(time_t now);
+time_t rep_hist_exit_stats_write(time_t now);
+void rep_hist_note_exit_bytes(uint16_t port, size_t num_written,
+ size_t num_read);
+void rep_hist_note_exit_stream_opened(uint16_t port);
+
+void rep_hist_buffer_stats_init(time_t now);
+void rep_hist_buffer_stats_add_circ(circuit_t *circ,
+ time_t end_of_interval);
+time_t rep_hist_buffer_stats_write(time_t now);
+void rep_hist_buffer_stats_term(void);
+void rep_hist_add_buffer_stats(double mean_num_cells_in_queue,
+ double mean_time_cells_in_queue, uint32_t processed_cells);
+char *rep_hist_format_buffer_stats(time_t now);
+void rep_hist_reset_buffer_stats(time_t now);
+
+void rep_hist_desc_stats_init(time_t now);
+void rep_hist_note_desc_served(const char * desc);
+void rep_hist_desc_stats_term(void);
+time_t rep_hist_desc_stats_write(time_t now);
+
+void rep_hist_conn_stats_init(time_t now);
+void rep_hist_note_or_conn_bytes(uint64_t conn_id, size_t num_read,
+ size_t num_written, time_t when);
+void rep_hist_reset_conn_stats(time_t now);
+char *rep_hist_format_conn_stats(time_t now);
+time_t rep_hist_conn_stats_write(time_t now);
+void rep_hist_conn_stats_term(void);
+
+void rep_hist_note_circuit_handshake_requested(uint16_t type);
+void rep_hist_note_circuit_handshake_assigned(uint16_t type);
+void rep_hist_log_circuit_handshake_stats(time_t now);
+
+void rep_hist_hs_stats_init(time_t now);
+void rep_hist_hs_stats_term(void);
+time_t rep_hist_hs_stats_write(time_t now);
+char *rep_hist_get_hs_stats_string(void);
+void rep_hist_seen_new_rp_cell(void);
+void rep_hist_stored_maybe_new_hs(const crypto_pk_t *pubkey);
+
+void rep_hist_free_all(void);
+
+void rep_hist_note_negotiated_link_proto(unsigned link_proto,
+ int started_here);
+void rep_hist_log_link_protocol_counts(void);
+
+extern uint64_t rephist_total_alloc;
+extern uint32_t rephist_total_num;
+#ifdef TOR_UNIT_TESTS
+extern int onion_handshakes_requested[MAX_ONION_HANDSHAKE_TYPE+1];
+extern int onion_handshakes_assigned[MAX_ONION_HANDSHAKE_TYPE+1];
+#endif
+
+/**
+ * Represents the type of a cell for padding accounting
+ */
+typedef enum padding_type_t {
+ /** A RELAY_DROP cell */
+ PADDING_TYPE_DROP,
+ /** A CELL_PADDING cell */
+ PADDING_TYPE_CELL,
+ /** Total counts of padding and non-padding together */
+ PADDING_TYPE_TOTAL,
+ /** Total cell counts for all padding-enabled channels */
+ PADDING_TYPE_ENABLED_TOTAL,
+ /** CELL_PADDING counts for all padding-enabled channels */
+ PADDING_TYPE_ENABLED_CELL
+} padding_type_t;
+
+/** The amount of time over which the padding cell counts were counted */
+#define REPHIST_CELL_PADDING_COUNTS_INTERVAL (24*60*60)
+void rep_hist_padding_count_read(padding_type_t type);
+void rep_hist_padding_count_write(padding_type_t type);
+char *rep_hist_get_padding_count_lines(void);
+void rep_hist_reset_padding_counts(void);
+void rep_hist_prep_published_padding_counts(time_t now);
+void rep_hist_padding_count_timers(uint64_t num_timers);
+
+#endif /* !defined(TOR_REPHIST_H) */
+
diff --cc src/test/test_router.c
index 613ec0402,51055a336..6e64131fc
--- a/src/test/test_router.c
+++ b/src/test/test_router.c
@@@ -4,114 -3,140 +4,236 @@@
/**
* \file test_router.c
- * \brief Unittests for code in src/or/router.c
+ * \brief Unittests for code in router.c
**/
-#include "or.h"
-#include "hibernate.h"
-#include "log_test_helpers.h"
-#include "main.h"
-#include "rephist.h"
-#include "router.h"
-#include "test.h"
+#include "core/or/or.h"
+#include "app/config/config.h"
++#include "core/mainloop/main.h"
++#include "feature/hibernate/hibernate.h"
++#include "feature/nodelist/routerinfo_st.h"
++#include "feature/nodelist/routerlist.h"
++#include "feature/relay/router.h"
++#include "feature/stats/rephist.h"
+#include "lib/crypt_ops/crypto_curve25519.h"
+#include "lib/crypt_ops/crypto_ed25519.h"
- #include "feature/relay/router.h"
- #include "feature/nodelist/routerlist.h"
-
- #include "feature/nodelist/routerinfo_st.h"
+
+/* Test suite stuff */
+#include "test/test.h"
++#include "test/log_test_helpers.h"
+
+NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+
+static routerinfo_t* mock_routerinfo;
+
+static const routerinfo_t*
+NS(router_get_my_routerinfo)(void)
+{
+ crypto_pk_t* ident_key;
+ crypto_pk_t* tap_key;
+ time_t now;
+
+ if (!mock_routerinfo) {
+ /* Mock the published timestamp, otherwise router_dump_router_to_string()
+ * will poop its pants. */
+ time(&now);
+
+ /* We'll need keys, or router_dump_router_to_string() would return NULL. */
+ ident_key = pk_generate(0);
+ tap_key = pk_generate(0);
+
+ tor_assert(ident_key != NULL);
+ tor_assert(tap_key != NULL);
+
+ mock_routerinfo = tor_malloc_zero(sizeof(routerinfo_t));
+ mock_routerinfo->nickname = tor_strdup("ConlonNancarrow");
+ mock_routerinfo->addr = 123456789;
+ mock_routerinfo->or_port = 443;
+ mock_routerinfo->platform = tor_strdup("unittest");
+ mock_routerinfo->cache_info.published_on = now;
+ mock_routerinfo->identity_pkey = crypto_pk_dup_key(ident_key);
+ router_set_rsa_onion_pkey(tap_key, &mock_routerinfo->onion_pkey,
+ &mock_routerinfo->onion_pkey_len);
+ mock_routerinfo->bandwidthrate = 9001;
+ mock_routerinfo->bandwidthburst = 9002;
+ }
+
+ return mock_routerinfo;
+}
+
+/* If no distribution option was set, then check_bridge_distribution_setting()
+ * should have set it to "any". */
+static void
+test_router_dump_router_to_string_no_bridge_distribution_method(void *arg)
+{
+ const char* needle = "bridge-distribution-request any";
+ or_options_t* options = get_options_mutable();
+ routerinfo_t* router = NULL;
+ curve25519_keypair_t ntor_keypair;
+ ed25519_keypair_t signing_keypair;
+ char* desc = NULL;
+ char* found = NULL;
+ (void)arg;
+
+ NS_MOCK(router_get_my_routerinfo);
+
+ options->ORPort_set = 1;
+ options->BridgeRelay = 1;
+
+ /* Generate keys which router_dump_router_to_string() expects to exist. */
+ tt_int_op(0, ==, curve25519_keypair_generate(&ntor_keypair, 0));
+ tt_int_op(0, ==, ed25519_keypair_generate(&signing_keypair, 0));
+
+ /* Set up part of our routerinfo_t so that we don't trigger any other
+ * assertions in router_dump_router_to_string(). */
+ router = (routerinfo_t*)router_get_my_routerinfo();
+ tt_ptr_op(router, !=, NULL);
+
+ router->onion_curve25519_pkey = &ntor_keypair.pubkey;
+
+ /* Generate our server descriptor and ensure that the substring
+ * "bridge-distribution-request any" occurs somewhere within it. */
+ crypto_pk_t *onion_pkey = router_get_rsa_onion_pkey(router->onion_pkey,
+ router->onion_pkey_len);
+ desc = router_dump_router_to_string(router,
+ router->identity_pkey,
+ onion_pkey,
+ &ntor_keypair,
+ &signing_keypair);
+ crypto_pk_free(onion_pkey);
+ tt_ptr_op(desc, !=, NULL);
+ found = strstr(desc, needle);
+ tt_ptr_op(found, !=, NULL);
+
+ done:
+ NS_UNMOCK(router_get_my_routerinfo);
+
+ tor_free(desc);
+}
+ static routerinfo_t *mock_router_get_my_routerinfo_result = NULL;
+
+ static const routerinfo_t *
+ mock_router_get_my_routerinfo(void)
+ {
+ return mock_router_get_my_routerinfo_result;
+ }
+
+ static long
+ mock_get_uptime_3h(void)
+ {
+ return 3*60*60;
+ }
+
+ static long
+ mock_get_uptime_1d(void)
+ {
+ return 24*60*60;
+ }
+
+ static int
+ mock_rep_hist_bandwidth_assess(void)
+ {
+ return 20001;
+ }
+
+ static int
+ mock_we_are_not_hibernating(void)
+ {
+ return 0;
+ }
+
+ static int
+ mock_we_are_hibernating(void)
+ {
+ return 0;
+ }
+
+ static void
+ test_router_check_descriptor_bandwidth_changed(void *arg)
+ {
+ (void)arg;
+ routerinfo_t routerinfo;
+ memset(&routerinfo, 0, sizeof(routerinfo));
+ mock_router_get_my_routerinfo_result = NULL;
+
+ MOCK(we_are_hibernating, mock_we_are_not_hibernating);
+ MOCK(router_get_my_routerinfo, mock_router_get_my_routerinfo);
+ mock_router_get_my_routerinfo_result = &routerinfo;
+
+ /* When uptime is less than 24h, no previous bandwidth, no last_changed
+ * Uptime: 10800, last_changed: 0, Previous bw: 0, Current bw: 0 */
+ routerinfo.bandwidthcapacity = 0;
+ MOCK(get_uptime, mock_get_uptime_3h);
+ setup_full_capture_of_logs(LOG_INFO);
+ check_descriptor_bandwidth_changed(time(NULL));
+ expect_log_msg_not_containing(
+ "Measured bandwidth has changed; rebuilding descriptor.");
+ teardown_capture_of_logs();
+
+ /* When uptime is less than 24h, previous bandwidth,
+ * last_changed more than 3h ago
+ * Uptime: 10800, last_changed: 0, Previous bw: 10000, Current bw: 0 */
+ routerinfo.bandwidthcapacity = 10000;
+ setup_full_capture_of_logs(LOG_INFO);
+ check_descriptor_bandwidth_changed(time(NULL));
+ expect_log_msg_containing(
+ "Measured bandwidth has changed; rebuilding descriptor.");
+ teardown_capture_of_logs();
+
+ /* When uptime is less than 24h, previous bandwidth,
+ * last_changed more than 3h ago, and hibernating
+ * Uptime: 10800, last_changed: 0, Previous bw: 10000, Current bw: 0 */
+
+ UNMOCK(we_are_hibernating);
+ MOCK(we_are_hibernating, mock_we_are_hibernating);
+ routerinfo.bandwidthcapacity = 10000;
+ setup_full_capture_of_logs(LOG_INFO);
+ check_descriptor_bandwidth_changed(time(NULL));
+ expect_log_msg_not_containing(
+ "Measured bandwidth has changed; rebuilding descriptor.");
+ teardown_capture_of_logs();
+ UNMOCK(we_are_hibernating);
+ MOCK(we_are_hibernating, mock_we_are_not_hibernating);
+
+ /* When uptime is less than 24h, last_changed is not more than 3h ago
+ * Uptime: 10800, last_changed: x, Previous bw: 10000, Current bw: 0 */
+ setup_full_capture_of_logs(LOG_INFO);
+ check_descriptor_bandwidth_changed(time(NULL));
+ expect_log_msg_not_containing(
+ "Measured bandwidth has changed; rebuilding descriptor.");
+ teardown_capture_of_logs();
+
+ /* When uptime is less than 24h and bandwidthcapacity does not change
+ * Uptime: 10800, last_changed: x, Previous bw: 10000, Current bw: 20001 */
+ MOCK(rep_hist_bandwidth_assess, mock_rep_hist_bandwidth_assess);
+ setup_full_capture_of_logs(LOG_INFO);
+ check_descriptor_bandwidth_changed(time(NULL) + 6*60*60 + 1);
+ expect_log_msg_containing(
+ "Measured bandwidth has changed; rebuilding descriptor.");
+ UNMOCK(get_uptime);
+ UNMOCK(rep_hist_bandwidth_assess);
+ teardown_capture_of_logs();
+
+ /* When uptime is more than 24h */
+ MOCK(get_uptime, mock_get_uptime_1d);
+ setup_full_capture_of_logs(LOG_INFO);
+ check_descriptor_bandwidth_changed(time(NULL));
+ expect_log_msg_not_containing(
+ "Measured bandwidth has changed; rebuilding descriptor.");
+ teardown_capture_of_logs();
+
+ done:
+ UNMOCK(get_uptime);
+ UNMOCK(router_get_my_routerinfo);
+ UNMOCK(we_are_hibernating);
+ }
+
#define ROUTER_TEST(name, flags) \
{ #name, test_router_ ## name, flags, NULL, NULL }
struct testcase_t router_tests[] = {
+ ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK),
+ ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK),
END_OF_TESTCASES
};
-
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits